{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分类训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 机器学习的HelloWorld ，一个新的分类算法，都会看看在MNIST的上的执行结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x00\\x00\\x00\\x03'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import struct\n",
    "struct.pack('>i',3) # 高位字节  打包"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#查看图片\n",
    "![jupyter](./g1.png)\n",
    "![jupyter](./g2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2051, 60000, 28, 28)\n",
      "47040000\n"
     ]
    }
   ],
   "source": [
    "import struct\n",
    "with open('./MNIST_data/train-images-idx3-ubyte', 'rb') as f:\n",
    "    buffer = f.read(4*4) # 4个int\n",
    "    head = struct.unpack('>iiii',buffer)  ##解析除4个int类型\n",
    "    print(head)\n",
    "    length = head[1] * head[2]  * head[3]  # 读图片 像素6w*28*28\n",
    "    print(length)\n",
    "    buffer = f.read(length)\n",
    "#   print(buffer)\n",
    "    data = struct.unpack('>{}B'.format(length),buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47040000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tuple"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)    ###元组类型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "imgs = np.reshape(data, (head[1], head[2], head[3]))  # 将数据 转换维度   所有的图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgs.shape   # 6w个图片 28行 28列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "for i in range(5):\n",
    "    plt.imshow(imgs[i], cmap = 'gray')\n",
    "    plt.show()\n",
    "    \n",
    "    # 展示前五章图片\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\feature_extraction\\text.py:17: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working\n",
      "  from collections import Mapping, defaultdict\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "# 读取数据集内容,   读取本地 mnist-original 在文件夹mldata下 mldata文件夹与当前文件 位于同一层目录\n",
    "mnist = fetch_mldata('mnist-original', data_home='./')  #如果是从网络上下载数据 把括号里改成一个完整的url即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DESCR': 'mldata.org dataset: mnist-original',\n",
       " 'COL_NAMES': ['label', 'data'],\n",
       " 'target': array([0., 0., 0., ..., 9., 9., 9.]),\n",
       " 'data': array([[0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0],\n",
       "        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* DESCR 数据集描述\n",
    "* data 包含一个数组，每个实例为一行，每个特征为一列\n",
    "* target 包含一个带有标签的数组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mnist['data'], mnist['target'] # 标签和训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape   # 7w  6w的训练 1w的测试，7w个 784个数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_digit = X[36000]\n",
    "some_digit_image = some_digit.reshape(28, 28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAANpElEQVR4nO3db6xU9Z3H8c9XLQ+UJoB3NCAutzZg1piUkgnZxE1jbbZRicE+qMIDZJMmtw/EQMSkpE2shiekrjY1MU3oQnpduzaYlgUj2a3BJoQHVkcDgiVFivyrN9wBEnv7gHSx3z64x+YCc37nMufMnIHv+5VMZuZ858z5Zrgfzsz5zZmfubsAXPuuq7sBAP1B2IEgCDsQBGEHgiDsQBA39HNjQ0NDPjw83M9NAqEcO3ZMZ86csU61UmE3s/sl/UTS9ZL+0903pR4/PDysVqtVZpMAEprNZm6t67fxZna9pJckPSDpLkkrzeyubp8PQG+V+cy+VNIRdz/q7n+V9EtJy6tpC0DVyoT9Nkknp9w/lS27iJmNmFnLzFrtdrvE5gCUUSbsnQ4CXPbdW3ff7O5Nd282Go0SmwNQRpmwn5J0+5T78yV9Uq4dAL1SJuzvSlpoZl8ysxmSVkjaWU1bAKrW9dCbu18wszWS/k+TQ29b3f3DyjoDUKlS4+zuvkvSrop6AdBDfF0WCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIErN4grUadu2bcn6gQMHcmsvv/xy1e1c5Pjx4z19/m6UCruZHZM0IekzSRfcvVlFUwCqV8We/evufqaC5wHQQ3xmB4IoG3aX9Bsze8/MRjo9wMxGzKxlZq12u11ycwC6VTbs97j7EkkPSHrczL526QPcfbO7N9292Wg0Sm4OQLdKhd3dP8muxyVtl7S0iqYAVK/rsJvZTWb2xc9vS/qmpINVNQagWmWOxt8qabuZff48/+3u/1tJV7hmTExM5Nb27t2bXHfjxo3J+ttvv52sZ3+byHQddnc/KukrFfYCoIcYegOCIOxAEIQdCIKwA0EQdiAITnG9xl24cCFZHxsbK/X8RcNjH3/8cW7trbfeKrXtXhoaGkrWV6xY0adOqsOeHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCYJz9Glc0jj48PJysu3uyPsinkS5evDi3tmrVquS6y5YtS9YXLlzYVU91Ys8OBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzn6Ne+qpp5L1onH0onqRefPm5dZGRjrOGPYPTz/9dKlt42Ls2YEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMbZrwFbt27Nre3atSu5btnz0YvWP3v2bG6t6DftDx8+nKwvWrQoWcfFCvfsZrbVzMbN7OCUZXPM7E0z+yi7nt3bNgGUNZ238T+XdP8lyzZI2u3uCyXtzu4DGGCFYXf3PZLOXbJ4uaTR7PaopIcr7gtAxbo9QHeru49JUnZ9S94DzWzEzFpm1mq3211uDkBZPT8a7+6b3b3p7s1Go9HrzQHI0W3YT5vZXEnKrserawlAL3Qb9p2SVme3V0vaUU07AHrFpvG74K9KulfSkKTTkn4o6X8kbZP0T5JOSPq2u196EO8yzWbTW61WyZbjSY2jS9KTTz6ZW5uYmCi17Tp/N37BggXJ+tGjR3u27atVs9lUq9Xq+I9S+KUad1+ZU/pGqa4A9BVflwWCIOxAEIQdCIKwA0EQdiAITnG9Cjz77LPJepnhtVmzZiXrM2fOTNavuy69vzh//nxubXw8/V2s48ePJ+u4MuzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtmvAsuXL0/WX3rppdza6tWrc2uStGbNmmR9yZIlyXqRsbGx3NqyZcuS6+7fv7/UtnEx9uxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7FeBF198sVS9Tqmfoi76meqiOq4Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIJx9szJkyeT9RtvvDG3dvPNN1fdzjUjdU560XTPRfUdO3Yk60W/AxBN4Z7dzLaa2biZHZyy7Bkz+5OZ7csuD/a2TQBlTedt/M8l3d9h+Y/dfXF22VVtWwCqVhh2d98j6VwfegHQQ2UO0K0xsw+yt/mz8x5kZiNm1jKzVrvdLrE5AGV0G/afSvqypMWSxiQ9n/dAd9/s7k13bzYajS43B6CsrsLu7qfd/TN3/5ukn0laWm1bAKrWVdjNbO6Uu9+SdDDvsQAGQ+E4u5m9KuleSUNmdkrSDyXda2aLJbmkY5K+28MeK7Fp06ZkfXR0NFmfMWNGbu2OO+5Irrt9+/Zk/Wp29uzZZH3Dhg25tYMH0/uI4eHhblpCjsKwu/vKDou39KAXAD3E12WBIAg7EARhB4Ig7EAQhB0IIswpru+8806yfvjw4a6f+8SJE8n6+vXrk/Xnn8/9AmLtik79feONN5L11PDaDTek//zuvvvuZJ1TWK8Me3YgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCCLMOHsvzZo1K1kf5HH0ImvXrk3Wi37OOWXevHk9e25cjj07EARhB4Ig7EAQhB0IgrADQRB2IAjCDgQRZpy96GeJZ86cmaxPTEzk1h566KFuWuqLRx99NFl/7bXXknV3T9aLplVOee6557peF1eOPTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBBFmnP2FF15I1o8cOZKsp34f/fz588l1i8ayi2zcuDFZ//TTT3Nr586dS65bNE5+5513JuuPPfZY1/U5c+Yk10W1CvfsZna7mf3WzA6Z2YdmtjZbPsfM3jSzj7Lr2b1vF0C3pvM2/oKk9e7+z5L+RdLjZnaXpA2Sdrv7Qkm7s/sABlRh2N19zN3fz25PSDok6TZJyyWNZg8blfRwr5oEUN4VHaAzs2FJX5X0O0m3uvuYNPkfgqRbctYZMbOWmbXa7Xa5bgF0bdphN7OZkn4laZ27/3m667n7Zndvunuz0Wh00yOACkwr7Gb2BU0G/Rfu/uts8Wkzm5vV50oa702LAKpQOPRmk2MzWyQdcvep41c7Ja2WtCm7vqp/93fdunXJempa5t27dyfX3bJlS7Ley9NIFy1alKwPDQ0l66+88kqyvmDBgivuCfWYzjj7PZJWSTpgZvuyZd/XZMi3mdl3JJ2Q9O3etAigCoVhd/e9kvJ2Ld+oth0AvcLXZYEgCDsQBGEHgiDsQBCEHQgizCmuRe67775kPTWWXnQa6f79+5P1PXv2JOuvv/56sv7EE0/k1h555JHkuvPnz0/Wce1gzw4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQVjRudRVajab3mq1+rY9IJpms6lWq9XxLFX27EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxBEYdjN7HYz+62ZHTKzD81sbbb8GTP7k5ntyy4P9r5dAN2aziQRFyStd/f3zeyLkt4zszez2o/d/T961x6AqkxnfvYxSWPZ7QkzOyTptl43BqBaV/SZ3cyGJX1V0u+yRWvM7AMz22pms3PWGTGzlpm12u12qWYBdG/aYTezmZJ+JWmdu/9Z0k8lfVnSYk3u+Z/vtJ67b3b3prs3G41GBS0D6Ma0wm5mX9Bk0H/h7r+WJHc/7e6fufvfJP1M0tLetQmgrOkcjTdJWyQdcvcXpiyfO+Vh35J0sPr2AFRlOkfj75G0StIBM9uXLfu+pJVmtliSSzom6bs96RBAJaZzNH6vpE6/Q72r+nYA9ArfoAOCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRh7t6/jZm1JR2fsmhI0pm+NXBlBrW3Qe1LorduVdnbAnfv+PtvfQ37ZRs3a7l7s7YGEga1t0HtS6K3bvWrN97GA0EQdiCIusO+uebtpwxqb4Pal0Rv3epLb7V+ZgfQP3Xv2QH0CWEHgqgl7GZ2v5n9wcyOmNmGOnrIY2bHzOxANg11q+ZetprZuJkdnLJsjpm9aWYfZdcd59irqbeBmMY7Mc14ra9d3dOf9/0zu5ldL+mwpH+TdErSu5JWuvvv+9pIDjM7Jqnp7rV/AcPMvibpL5Jedve7s2U/knTO3Tdl/1HOdvfvDUhvz0j6S93TeGezFc2dOs24pIcl/btqfO0SfT2iPrxudezZl0o64u5H3f2vkn4paXkNfQw8d98j6dwli5dLGs1uj2ryj6XvcnobCO4+5u7vZ7cnJH0+zXitr12ir76oI+y3STo55f4pDdZ87y7pN2b2npmN1N1MB7e6+5g0+ccj6Zaa+7lU4TTe/XTJNOMD89p1M/15WXWEvdNUUoM0/nePuy+R9ICkx7O3q5ieaU3j3S8dphkfCN1Of15WHWE/Jen2KffnS/qkhj46cvdPsutxSds1eFNRn/58Bt3serzmfv5hkKbx7jTNuAbgtatz+vM6wv6upIVm9iUzmyFphaSdNfRxGTO7KTtwIjO7SdI3NXhTUe+UtDq7vVrSjhp7ucigTOOdN824an7tap/+3N37fpH0oCaPyP9R0g/q6CGnrzsk7c8uH9bdm6RXNfm27v81+Y7oO5JulrRb0kfZ9ZwB6u2/JB2Q9IEmgzW3pt7+VZMfDT+QtC+7PFj3a5foqy+vG1+XBYLgG3RAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EMTfAa5yOtysgto/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit_image, cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[22000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 建立测试集和训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练集 和 测试集 训练标签，测试标签\n",
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([10198, 19977, 19232, ..., 39046, 55494, 51721])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将数据集合交叉洗牌，交叉验证时，每个子集合数据分布均匀，有些机器学习算法对训练实例的顺序敏感\n",
    "import numpy as np\n",
    "shuffle_index = np.random.permutation(60000)  # 6w下标，随机排列\n",
    "shuffle_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练集和训练标签洗牌，然后赋值\n",
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       ...,\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0],\n",
       "       [0, 0, 0, ..., 0, 0, 0]], dtype=uint8)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练一个二元分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 识别数字5 ，二元分类5或者非5\n",
    "# 创建目标向量\n",
    "y_train_5 = (y_train == 5)   #y_train标签等于5就为true 否则为false\n",
    "y_train_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, ...,  True, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       ...,\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [ True, False, False, ..., False, False, False]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_5.reshape(20, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_5 = (y_test == 5) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SGD 梯度下降 分类器， 适合非常大的数据集，独立处理训练集数据，一次一个，适合在线学习，\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "# 随机种子，每次的值一直\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train, y_train_5)\n",
    "\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 性能考核"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用交叉验证测量精度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.9678 , 0.96635, 0.96445])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 评估分类器比评估回归器要困难得多\n",
    "\n",
    "# 3个折叠，正确率达到 95% 以上\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把每张图都分类成 非5\n",
    "from sklearn.base import BaseEstimator\n",
    "class Never5Classifier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False],\n",
       "       [False],\n",
       "       [False],\n",
       "       ...,\n",
       "       [False],\n",
       "       [False],\n",
       "       [False]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.zeros((len(X), 1), dtype=bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90795, 0.91125, 0.90975])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 分类器，\n",
    "never_5_clf = Never5Classifier()\n",
    "# 交叉验证\n",
    "cross_val_score(never_5_clf, X_train, y_train_5, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 准确率超过90% ，因为5的图像大约只有10%，你猜一张图不是5， 90%的时间你都是正确的\n",
    "* 这说明准确率无法成为分类器的首要性能指标，特别是当你处理偏科数据集， 某些类比其他类更为频繁"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# 评估分类器性能的更好方法是混淆矩阵\n",
    "# A类别实例被分为B类别次数\n",
    "# 想要知道分类器将数字3和数字5混淆多少次，通过混淆矩阵的5行3列\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "# 预测结果\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 与 cross_val_score 相比\n",
    "* 同样执行交叉验证\n",
    "* 返回的不是评估分数，是每个折叠的预测\n",
    "* 每一个实例在模型预测时使用的数据，在训练期间从未见过"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54074,   505],\n",
       "       [ 1523,  3898]], dtype=int64)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "# 混淆\n",
    "confusion_matrix(y_train_5, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 行表示实际类别，列表示预测类别\n",
    "# 第一行 第一列 53272 被正确的分为 非5 ，真负类\n",
    "# 第一行 第二列 1307 被错误的分类成 5 ，假正类\n",
    "# 第二行 第一列 1077 张被错误的分为 非5， 假负类\n",
    "# 第二行 第二列 4344 张被正确的分在了5 ，真正类\n",
    "# 这种衡量方式太复杂，我们可以用更简单的指标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54579,     0],\n",
       "       [    0,  5421]], dtype=int64)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_perfect_predictions = y_train_5   # 实际的预测标签，完美预测  实际==预测\n",
    "confusion_matrix(y_train_5, y_train_perfect_predictions)   ##混淆矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "\n",
    "## 正类预测的准确率 被称为分类器的精度\n",
    "\n",
    "\n",
    "$\n",
    "\\text{精度} = \\cfrac{TP}{TP + FP}\n",
    "$\n",
    "\n",
    "TP是真正类的数量，FP是假正类的数量\n",
    "\n",
    "\n",
    "\n",
    "$\n",
    "\\text{召回率TPR} = \\cfrac{TP}{TP + FN}\n",
    "$\n",
    "* 检测正类实例的比例\n",
    "FN是假负类的数量\n",
    "![jupyter](./zhaohui.jpg)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精度和召回率\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8853054735407676"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "# 实际标签和交叉验证的预测标签，获得精度\n",
    "precision_score(y_train_5, y_train_pred) # 4327 / 4327 + 1276"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7190555248109205"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred)    #  4327 / 4327 + 1094"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 说明 检测一张图的时候，只有90%的概率是准确的，而且只有64%的数字5 被它检测出来\n",
    "# 精度和召回率合成单一指标，成为 F1 分数，谐波平均值\n",
    "# 平均值平等对待所有的值，谐波平均值会给予较低值更高的权重，只有召回率和精度都很高时，才能获得较高的F1分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\n",
    "F_1 = \\cfrac{2}{\\cfrac{1}{\\text{precision}} + \\cfrac{1}{\\text{recall}}} = 2 \\times \\cfrac{\\text{precision}\\, \\times \\, \\text{recall}}{\\text{precision}\\, + \\, \\text{recall}} = \\cfrac{TP}{TP + \\cfrac{FN + FP}{2}}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7935667752442996"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_5, y_train_pred)   # f1分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# F1分数对那些具有相近精度和召回率 分类器更有利，这不一定符合你的期望\n",
    "# 有时候你更关心精度，有时你能关心召回率\n",
    "# 训练一个分类器检测儿童可以放心观看的视频，你可能要求拦截了很多好的视频，低召回率，保留下来的都是安全的视频，高精度\n",
    "# 不能同时增加精度并减少召回率，反之亦然"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精度/召回率权衡"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "![jupyter](./quanheng.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* SGDClassifier对每个实例基于决策函数计算一个分值，大于阀值为正类，否则为负类\n",
    "* 中间阀值右侧找到4个真正类 真5 ， 一个假正类 6， 精度为 4/5 80%\n",
    "* 在所有的6个 真正的5 中，分类器找到了4个，召回率为 4/6 67%\n",
    "* 提高阀值，向右移动，精度提高，召回降低\n",
    "* 反之阀值降低，召回提高，精度降低\n",
    "* SKlearn不可以直接设置阀值，可以访问决策分数，\n",
    "* SGDClassifier 默认阀值为0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如何设置阀值\n",
    "\n",
    "# 用predict_proba得到每个实例属于正类的概率，然后对概率切一下。以LogisticRegression为例\n",
    "# clf = LogisticRegression()\n",
    "# clf.fit(X_train, y_train)\n",
    "# pred_proba = clf.predict_proba(X_test)[:, 1]\n",
    "# threshold = 0.75  # 阀值设置为0.75\n",
    "# pred_label = pred_proba > threshold\n",
    "\n",
    "# pred_proba是每个实例为真的概率\n",
    "# 假设阈值是0.75\n",
    "# pred_label里True就是概率大于0.75的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([45693.37296594])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 返回决策值decision_function\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0  # 默认阈值为0\n",
    "y_some_digit_pred = (y_scores > threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提高阀值可以降低召回率，提高阀值到200000，就错了这个图\n",
    "threshold = 200000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何决定使用什么阀值\n",
    "### 返回决策值，而不是预测结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_5, cv=3,\n",
    "                             method=\"decision_function\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 有了y_scores，可以计算所有可能的阀值的精度和召回率\n",
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "# 计算 精度 召回 阈值\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEPCAYAAABx8azBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd1xWZf/A8c/FniKiuEBFxT0TR2aUe4/MNDVXPY+ren5tGz4tW6YtSx9XjjT3ytTSTNMyF25TUXIgaCKCgOwbrt8fR5aiogLnBr7vXud1n/uc6z7ne5DuL9c511Baa4QQQghhHhuzAxBCCCFKOknGQgghhMkkGQshhBAmk2QshBBCmEySsRBCCGEyScZCCCGEye6YjJVSc5RSEUqpo7fYr5RSU5RSIUqpw0qpB/I/TCGEEKL4ykvNeB7Q5Tb7uwL+15eRwP/uPywhhBCi5LhjMtZabweiblOkN/CdNuwCSiulKuZXgEIIIURxZ5cPx6gMnM/2Puz6tos3FlRKjcSoPePq6tqsTp06+XB6IYQQ1iYxEZKSwMbm5sXR8ebyEREQE2PsVwpcXKBCBWOf1nDxorE9Y3/Guo0NuLqCg4NRNjUVLJabz6lU4V17RAScv54Vvb3B19dYj4uDkyf3RWqty934mfxIxrldYq5jbGqtZwIzATxreOrq46pja2OLjbLBVtka69hga2OLi70LDrYO2Cpb7GzssLWxxVbZ4mTnhJuDGw62DjjbO1PBrQJlnMvg5uCGvY09djZ22Nva4+nkib2tfT5cnhBCFH3//ANffAGxsbB+PXz4IQwZYuw7eBAWLwZ3dyhdGjw8jNdSpYxE17Qp2Noaie6tt4xtU6fC5cvg6QnOzlCunHEcMM7h4XHrWK5dM44B0KULHDhgJLDsmjeHjRuN9StXoGzZWx9v+XLo189Y//hjePPNm8vY2RnXFxZmJHqAsWMhPj7rej08spaGDeFe64tPPw1z58Izz8Do0RAQYGy/dAkqVFDncvtMfiTjMMA323sf4MKdPnQ18SrLjy3Ph9Pfmqu9K64Orrg5uOVI+BmvTnZOuNi7YGdjl2PxdPLE3cEdd0d34w+BbH8Q2NnY4WrviqOdo5H4bexxsnPCw8kDF3sXPJ08cbJzwtXBFQ9HD1Rh/jkmhChUFgvs32980dvbQ3CwsSQlGQnqpZfAyckoGxRkbHd3Bzc3Y3F3NxKNs3NWsklPN2qCtra3P3dsLERFwerVRi3Uw8NIjKVLQ9264OcHmzfD8OGQkADR0Tk/v3BhVjI+fBg+/fTW54qPNxLYtWswaVLOfdHRxpKSkrXN3R18fIzE162bcf7si7NzVtmIiJyJeMUK41je3lnbHB3hvfeMn19yctaS8b5KlayyXl7QoEHO88XHG/9WlStnJWKAVauMBJmbt982zgmwbh0MGmQcO2MpW9ZYKlWC//u/rH/nK1fg0CFjfejQrEQMUL78rX/GKi8TRSilqgHrtNYNctnXHXgO6Aa0BKZorVvc6Zg1GtTQHy35iHSdTppOIy09LXM9NS2V+NR40tLTMvdZ0i2k6TSupVwj2ZJMSloKkYmRRCVGEZUYRZIlCUu6hdS0VJIsSUQnRd8phALnbOeMl4sXLvYueLt6U8WjCt4u3ng6X0/Y9q6UdSmLt6s3pZ1K4+/lj5uDm9lhCyGuCwqCefOMGt+ePVC9unE7dMMGI9lcvpwzadwoMTHrS7pNG9ixI/dyr70GEycaSWjoUFi61EiuZcoYCXb/fuNW54oV0OL6t+vEifD667kf79134Z13jBrjr7/CjBlZ+/r0Mf54GDcuK1EcOWJcU0yMsVy9aiyxsUYy27vXuNUbHw/ffGO8xsdDq1bQrp1xnampxh8AGbZtg8aNjT8ObicpKet8VarkTJb5KTXVWLIf/8cfjeR59erN1z1wYFZte8EC49/ldteQceu9dWvYudNYv3LF+DfMTim1T2sdwA3uWDNWSi0GHgXKKqXCgHcAewCt9XRgA0YiDgESgBF3OiaAp5MnAxoMyEvRe5KRlGOSY0iyJOVI9hlJPj4lnpS0FCzplswl0ZJIbHIsMUkxJKQmZP4RkPEHQWp6KtdSrmWup6alEpcSR3xKPHEpccQlx2WeNyE1gbDYMABOXjmZp7jdHNxwc3DDw9EDZ3tnfEr54F/Gn9JOpfEt5Us513JUdKtI5VKVKe9aXmreQtylRYuM53mXLhlfurt3w9GjUKMGLFsGD1zvnPnyy/D55zk/GxxsvGbUYZQybqdmfNGnpMCpU/D880bNN/uz0UaNjM9du2Y8O7x2zVgSE7PK2dhA9+5GMs5IjGfOGPvOnzcS6VdfwRNPGNs8PY1aad268MgjRiKJjs66vfrEE1CxIgwebPzx4O2ddXs4u4YNjeVOXF2NJJ4XjzySt3JOTsaz4YznwwXF3t5YsuvZM2+ffeop6NXLSK6RkVmvly7BuXM5/50dHY2E37nzzYn4dvJUMy4IAQEBOigoyJRzFwatNTHJMcQmxxKfEk9YbBhnr54lOik6R8KOSowiIj6CyIRITl45SZpOy/M5PJ088fP0o4xzGbxdvangWgFne2fsbexxsHWglGMpXOxdcHVwpbRTaZzsnKhWuho+pXywUTLeiyheLl0ykuq5c0aSTEkxbmFeuAAdO0L79kY5X1/j9mlufv/dqMECPPsszJ9v1AABPvsMmjUzEmDdujd/sd8rrXM2LoqJMW7jxscbt6GjoozknXF71Mcnf84rzHGrmrEkYytiSbeQkJrA1aSrxKfEE5Mcw/HLx7mSeIUrCVcIjQ0lIj6CsNgwLsRdIDY59p7P5WTnhIej8Zzbyc4JT2dPSjmWoqZnTfw8/ajlVQsvZy/cHNwo41wGd0d33B3cpSYuCpXFYjx/W70adu0yElNoaFZNNDg4q6Wqvz+EhOR+nK+/hueeM9affda41VyqlFFr1Nq4PermZtSK3d2NcpGRxnpuLX+FuFf3fJtaFB47GztKOZailGOpzG2tfFrlWlZrTWhMKBevXSQmKYaL1y5y6dqlzFvn2W/RxyTHEJ8ST3RSNGevniUyIZIkSxJJlqS7is9G2dzUSK2iW0VKO5XG3cEdB1sHHGwdaFi+Ic0rNaeie0Wc7JxwsnOSmri4rbQ0oxHRjh2wfbtxuxiMmm3ATV9bWbLXThs2NBoClS4Nbdsaydrd3ajJNm+eVW7q1LzFdLvWu0LkN6tOxjExMURGRpKSvZmeyMHj+n9VHKtAHv+C12i01qTr9BzraToNS5olsxFdxv50nW6s5/UuSiKc//s85693P1dKZXY7c7RzxNnO+bbdzhwcHChbtiwet+sbIYq89evh/fezWiRn8PY2Gku1aGE8Q/X1NdYbNID69Y11Dw+j1bGnZ9bnVq0q/GsQIr9YbTJOSkri0qVL+Pj44OzsLLdHrYDWOrMxm9aa1PRUI4GnWzITekYjuITUhMxGc/qGbucppJBuk46jrSMeTh64Objhau+KrY0tWmsSExMJCwvD0dERp4ymqKJI0tqo7f78s1FrfeUVqFXL2PfWW1ldQDJ4e8PIkVC1qvHe2dm4LS1EcWe1yfjy5cuUK1cOl4Jq5y7umlIKO2X0xQZwIm+JMiUthRRLCklpScQkxRCTHJPZej0+1Wgdo1C42LvgbO+Mu4M7pTxLERERQZXsHQhFkZGcDNOnG91gsj/HHTQoKxk3bw4PPmjUcN97r+Bb0wphzaw2GSclJVFB/u8sFjKeJbvhRlmXsmitSUlLIdGSSHRidGZNOj41nvjUeCITIiENzpw7w8OrHqZl5Za0rdaWAQ0GUMb5LvoKCFN8/73RFSSDvb1R0x0wwGhklWHWrMKPTQhrZbXJ2GKxYGdnteGJ+6CUwtHOEUc7R0o7GSMCZDQ6i0uJIyE1gbjkOFztXAmNCSU0JpTlx5bz8qaXCawaSM0yNantVZtetXtRtXRVk69GpKcb3YgynigcP561b/p0Y2jA/OoGJERxZbVdm44fP07dunULMSJhTbTWHDt+jAjnCIIuBLH+1Hq2ndt2U7lyLuVoXKExPWv15CHfh2hasam03C4kWsOLLxqDUPz73zBzprF9927j+XCPHoU7OL8QRUGR62csyVjc+Dtw/PJxjkQcITw2nN9Df+fnkJ9JtCTm+Exl98p0rdmVwKqBdKjegYruMptnftPaGJ/4xpGYTpyA2rXNiUmIokKSsShy7vQ7YEm3EB4bzq9nfmXLmS1sPbuVC3FZc5TYKBse8n2IzjU6MypgFGVdpOPo/fq//zP66aZlGyiuc2dYuzZrCjshxK3dKhnL/bxCNG/ePJRSmYu7uzuNGzfmm2++wWKxFEoMZ8+eRSnFvHnz8vyZjLjPnj1bYHHdCzsbO6qWrsrTTZ9mYd+FhL4QypahW5jQdgIdq3ckXafze+jvjN86nkqfVWLkjyM5GnE07/2lBUlJxtR7GapVy0rEPXoYYyH//LMkYiHul7SQMsHy5cvx8fEhNjaW5cuX8/zzzxMREcH7779f4OeuWLEiO3fupEaNGnn+TPfu3dm5cycVK1r3LV9bG1va+rWlrV9bAMJiw9h+bjuz989m69mtzNo/i1n7Z1HVoyq9a/dmYMOBtxzhrKRLSjJuQ8+bZ4zrvGSJMdPP6NFGTbhuXXkeLER+ktvUhWjevHmMGDGCU6dOUbNmzcztbdu2Zd++fcTG3jzWdGpqKnZ2diVy0JP8/B3YE76H97a9x9YzW3M8Z67hWYMO1TvQp04fHqn6CM72zrc5SvGXlgaTJ+ecmq9iRWPi+bzOwiOEuDW5TW3FmjdvTlxcHHv27EEpxbRp03jttdeoVKkSjo6OXL16FYBVq1bRqlUrXFxcKF26NE888QShuQxPNGvWLB544AGcnZ3x9PTkkUce4c8//wRyv029d+9eOnbsiJeXFy4uLlSvXp2xY8dm7s/tNnVqairjx4+nWrVqODg4UK1aNcaPH09qampmmYxzzZgxg7fffpuKFStSunRpevbsSditps0pIC0qt2D9oPXEvB7Dxqc2MqrZKBxtHfk7+m9m7JtB1++7UvXLqnz0+0dcS7lWqLFZi5MnjfGYsyfiadOMWY8kEQtRsIpkMlbq1ktG9wow1m9XNrtmzW5dbuTIrHL79uX/9Zw5cwZbW1vc3NwA+PDDDzl58iQzZ85k9erVODk5MX36dB5//HHq1avHihUrmDFjBkePHuWRRx4hLi4u81ivvPIKI0eO5IEHHmDZsmUsXLiQwMDAXJM2wLVr1+jcuTO2trbMmzePDRs28Pbbb9/xGfawYcP45JNPGDp0KOvWrWPEiBFMnDiRYcOG3VT2448/JiQkhDlz5vDVV1+xc+dOBg8efB8/sXtnb2tPpxqdmN5jOtHjovllyC+Mbjaa2l61uZxwmbe2vIX7x+70WtyLX/7+pUQ9X54/33gGDPDRR8ZEC2PGmBuTECWG1tqUpVmzZvp2jh07dst9RueK3JcZM7LKzZhx+7LZPfDArcv9+99Z5YKCbhv2bc2dO1cD+sSJEzo1NVVHRUXp6dOnaxsbG927d2995swZDeimTZvq9PT0zM/FxcXpUqVK6REjRuQ43pkzZ7S9vb3+4osvtNZanzp1StvY2OgXX3zxljFknGPu3Llaa6337t2rAX3o0KE7xn3mzBmttdZHjhzRgH7nnXdylJswYUKOY2WcKzAwMEe5SZMmaUCHh4ff9ud1u9+B/Jaenq5XHlup60+tr3mXzKX79911cGRwocVRmNLTtV682HjVWuvNm7V++mmtDx82Ny4hijMgSOeSE4tkzfh2KTZ7LXbkyNuXzW7fvluXy17bbtbs/uOvU6cO9vb2lClThrFjxzJ48GDmzJmTub9Pnz45nhHv3LmT2NhYBg8ejMViyVx8fHyoU6cO27dvB2Dz5s2kp6czMvsP4Q78/f0pXbo0o0aNYuHChZw/f/6On8k431PZxzzM9n7btpyDc3Tv3j3H+4YNGwLcsrZuBqUUfev25ejYo/z9n7956+G3cLR1ZP2p9dT+pjZt5rThROQJs8PMNxaLMWTlwIGwbp3xe96+PXz7rTEVoRCicBXJZFzUrV69mr1793LixAni4+P57rvvKFMma8zlG1stR0REANChQwfs7e1zLEeOHOHKlSsAma8+Pj55jsXDw4OtW7dSqVIlxo4dS5UqVWjQoAErV6685WeioqJyjTNjLPGM/RmyXxuA4/XZ2pOS7m4+5cJS3bM6H7T7gCNjjjCwwUAAdpzfQePpjXntl9fYfm47KWlFd1rPqCijUdaiRcYwldHR0jJaCLNJMjZBgwYNCAgIoHbt2rlOEXhjy2kvLy/AaEi1d+/em5aZ16vuZa/Phh4eHn5X8TRp0oSVK1cSFRWV2e2pf//+HD16NNfyGcn1n+wdULO9z4i3qPP38mfR44sIfi6Yx+o8RkpaCpP+nMQj8x6h8ueVmbVvVpFLyqtXG32FIyON94sXw9ChpoYkhECScZHQunVr3N3dCQkJISAg4Kal9vUxCDt06ICNjU1mcr5bdnZ2tGrVigkTJpCens7x7CP+Z/PI9aa1S5YsybH9+++/ByAwMPCezm+tannVYkX/FWwYtIFhjYdR2b0ykQmRjFw3Eu9J3jy/4XmiE6PNDvOOtm2Dvn0ho73fli3w+OPmxiSEMMigH0VAqVKlmDRpEs8++yyXL1+ma9eueHh4EB4ezrZt23j00UcZNGgQNWrU4MUXX+Tzzz8nLi6OXr16YWtry549e6hTpw4DBgy46djr1q1j5syZ9OnTBz8/P+Lj45kyZQru7u48+OCDucZTv359Bg4cyLvvvovFYqF169bs3LmTCRMmMHDgQBo1alTQP5JCZ6Ns6Orfla7+XdFaM+fAHCbumMipqFN8s/cbVh5fyacdP+XRao/iUyrvjwkKU8YNl4YN4eBBsJE/xYWwGpKMi4hRo0bh6+vLpEmTWLRoEampqVSuXJnAwECaNGmSWW7y5MnUrFmTadOmMX/+fFxdXWnUqBGdOnXK9bj+/v44OzszYcIELl68iLu7O82bN+eXX3657bPn+fPnU716debMmcMHH3xApUqVGDduHO+8806+X7u1UUrxzAPP8HTTp9l+bjtj1o/heORxhqweAkCdsnV4O/BtnmzwpFUN1tKmDbRrBz/9JIlYCGsjI3AJq1VUfgdS0lKYtW8WS/5awt7wvSSnJQPQpEITlvVbhr+Xv2mxbd9utJSWQTuEsA4yApcQBcTB1oFnWzzL7yN+5+rrV5nYYSJuDm4c/OcgzWc1Z/7B+abENXmykYR79TKmNxRCWC9JxkLkIyc7J1576DUuvHSBrjW7EpMcw/AfhtNqdit+Pf1rocWxbBm8+qqx3rcvVK9eaKcWQtwDScZCFAB3R3fWD1rP5I6TcbJzYnf4bjos6MDQ1UNzzLlcEE6ehIy2egMGwNy5MsWhENZOkrEQBUQpxcutX+bcC+d4oeUL2NnYseDwAnw+9+GVTa+Qlp6W7+eMjDTmGc5wvbeZEMLKSTIWooB5u3rzRZcvODjqIH3q9EGj+WznZzw05yHOx9x5+NG7MXIknDoFjRsbkz7Y2ubr4YUQBUSSsRCFpL53fVYPWM3Gpzbi5uDG7vDdtJnbhu3ntufbOT78EAICYP168PDIt8MKIQqYJGMhClmnGp04MOoAtb1qExoTyqPzHuWVTa/c89CaFgukXb/jXbcu7NkDlSvnY8BCiAInyVgIE9QsU5P9o/bz7wf+nXnbuvW3rfkr4q+7Ok5qKri7g50dJCQY26xonBEhRB5JMhbCJC72LszsOZNNT22iknsl9l3cR7OZzZh3cF6ej/Haa5Ax+dUN83YIIYoQScZCmKxjjY7sH7mfbv7dSE5LZsQPIxi8ajCJqYm3/dyUKfDll8b6Tz9JX2IhijJJxoVo3rx5KKUyFwcHB2rUqMGbb75p6ty+w4cPp1q1apnvz549i1KKefPmmRZTSVPerTzrB61narep2NvYs+jIIupOrUtwZPBNZS0WePll+L//M95PmwZduhRywEKIfCUTRZhg+fLl+Pj4EBcXx+rVq/n444+Ji4vj66+/Njs0YbKxzcfStEJTBq4cyLmYc7SY3YJNT22ipU/LzDKtWsG+fcb6O+/AmDEmBSuEyDd5qhkrpboopYKVUiFKqddz2e+hlPpRKXVIKfWXUmpE/odafDRp0oRWrVrRsWNHpk2bRocOHfj2229JT083OzRhBR70fZCjY4/ySNVHiE2Opc3cNmwM2QhAejr4X593YulSePdd8+IUQuSfOyZjpZQtMBXoCtQDBiql6t1Q7FngmNa6MfAo8JlSSgbgy6MHHniAxMREIiMjAUhISGDcuHH4+fnh4OCAn58fH3744U3J+vLly4wdOxZfX18cHR3x9fVlyJAhJCcbswaFhIQwZMgQ/Pz8cHZ2pnr16owZM4bo6OhCv0Zxd9wc3Php8E88Wu1RLOkWeizuwZwDc1BKs3AhbNsG/fubHaUQIr/k5TZ1CyBEa30aQCm1BOgNHMtWRgPuypi81Q2IAiz5HCsA6j3r6Leh38m/qSfPnj2Lh4cHXl5eWCwWOnfuzLFjx/jvf/9Lw4YN2bVrFxMmTCAqKorPPvsMgOjoaFq3bk1UVBTjx4+nUaNGRERE8MMPP5CSkoKjoyMXLlzAx8eHL7/8Ek9PT06fPs1HH31Et27d2LlzZ77FLwqGs70zG5/ayJh1Y5hzcA7PrH2GoAtBfN31awIDZWgtIYqTvCTjykD2MfvCgJY3lPkGWAtcANyBAVrrm+65KqVGAiMBqlSpci/xFgtpaWlYLJbMZ8YrV67kyy+/xNbWlgULFvDHH3+wbds2AgMDAWjfvj0A7733HuPGjcPb25svvviC06dPExQURNOmTTOPPXDgwMz1wMDAzGMAtG7dmpo1a/Lwww9z4MCBHJ8T1snB1oHJj8xk86IGhNZ+g/8F/Y/jkceZ32c+VTxK7v9DQhQ3eUnGuVVFb6wWdgYOAu2AGsAvSqnftdaxOT6k9UxgJkBAQMA9VS3zs0Zqljp16uR4P3bsWJ577jkAfv75Z6pWrUrr1q2xWLJuLnTq1Inx48eza9cuevXqxaZNm2jevPltE2pKSgqTJ0/mu+++49y5czlabAcHB0syLgISE6FPb1tCt79I+QcbYOk9mN/O/kbD/zVk85DNNK/c3OwQhRD5IC8NuMIA32zvfTBqwNmNAFZpQwhwBqiDyNXq1avZu3cvGzZsoEOHDkybNo3vvvsOgIiICM6dO4e9vX2OpUWLFgBcuXIl89XHx+e253njjTd49913eeqpp1i/fj179uxh1apVAKZ2pRJ59+yzsH27Mbzlnws7cuy5o3T3705sciy9lvTK94kmhBDmyEvNeC/gr5TyA8KBJ4FBN5QJBdoDvyulygO1gdP5GWhx0qBBA2rWrAlAu3btaNSoEa+++iqPP/44Xl5e+Pn5sWzZslw/m9EfuGzZsoSHh9/2PEuWLGHo0KGMHz8+c9u1a9fy5yJEgbJYjH7Ec+ca7xctyhjUw5tVA1bReWFnfjv7G90WdeOXIb9Qwa2CmeEKIe7THWvGWmsL8BywETgOLNNa/6WUGq2UGn292ASgtVLqCPArME5rHVlQQRcnjo6OTJo0iYiICKZNm0aXLl04f/48bm5uBAQE3LSULVsWMG5b79mzh0OHDt3y2AkJCdjb2+fYNjfj211YtW7djME8bGyM7kvZHv3jYOvAyv4r8S/jz9GIowTODeTs1bNmhSqEyAd5GvRDa70B2HDDtunZ1i8AnfI3tJKjV69eNG/enMmTJxMSEsLcuXNp3749L7/8Mo0bNyYlJYW///6btWvXsmbNGlxcXHjxxRdZtGgRHTp0YPz48TRs2JDIyEh++OEHpk+fjru7O126dGH+/Pk0bNiQmjVrsmrVKv7880+zL1fkwQsvwC+/GMNcdsrl/6wyzmXYNGQTD377IKeiTtFqdisOjDpARfeKhR+sEOK+yQhcVuKDDz6gc+fOzJ49m40bN/LJJ58wc+ZMzpw5g6urKzVq1KB79+44OBjdt0uXLs2OHTsYP348n3zyCVeuXKF8+fK0a9cus8zXX3+N1pq33noLgG7durF48eLM58/CenXpAiEhUKPGrctUK12Ng6MO0n1Rd/Zd3Mfjyx5n67CtONo5Fl6gQoh8obQ2p3VyQECADgoKuuX+48ePU7du3UKMSFibkvY7cPQoTJ1q3J6+m2kQI+IjaDazGWGxYbSt1pafn/oZB1sZc0cIa6SU2qe1Drhxu0wUIYQVSEyEhg1h+nSYOfPuPuvt6s3aJ9dS3rU8W89upf/y/lxJuFIwgQohCoQkYyGswOvZRnzv2/fuP9+0YlPWDlyLvY09PwT/gN9XfqwNXpt/AQohCpQkYyFMNnmyMTcxwLp1UK7cvR2nReUWHBh1gPZ+7YlLiaP/8v7sCtuVf4EKIQqMJGMhTBQUBK++aqyPHw/du9/f8ep71+eXIb8wqOEgktOS6fZ9N3ael3HIhbB2Vp2MzWpcJsxXEv7tL1+Gxx4z1l1c8m86RKUUc3rNoZ1fO6KToum+qDtnos/kz8GFEAXCapOxvb09iYmJZochTJKYmHjTgCXFUZ060Lo1REeDbT5OxORo58jPg3+mS80uRCdF03dZX+JT4vPvBEKIfGW1ydjb25vw8HASEhJKRC1JGLTWJCQkEB4ejre3t9nhFKhy5eDnn43nxA4F0BPJ3taexY8vpqpHVQ7+c5C+y/oSmSAD4wlhjay2nzFAbGwsERERpKamFlJUwhrY29vj7e1NqVKlzA6lQISGQvny4FhIY3Psv7iftvPbEpscS/1y9dk+YjtlnMsUzsmFEDncqp+xVSdjIYqbuDho3NiYhWnNGvDyKpzznrpyip6LexJ8JZhaXrX4fcTveLsW7zsPQlgjGfRDCCvw1Vdw5gxcuwZOToV3Xn8vfzYM3oB/GX9OXjlJmzltOBpxtPACEELcliRjIQrJvn3w4YfG+qRJ4OpauOev7lmd34b/RpMKTTgVdYoHv32Qw5cOF24QQohcSTIWohDEx0OPHpCUBMOHQ4cO5sRRyb0Sv4/4nRaVW3At5Rrtv2vPsYthRFkAACAASURBVMvHzAlGCJFJkrEQheCrr+Cff4xZmKZPv3P5guTm4MbPg3+mQ/UORCZE0nRGU0KiQswNSogSTpKxEAUsLAyuz2LJ118XXivq2/F09mTtk2tpXqk5KWkpPDTnIQ7+c9DssIQosSQZC1HAKlaE77+HQYOga1ezo8nibO/Mxqc20rF6RyLiI+i9pLf0QxbCJJKMhShgNjbg42PcqrY2ns6e/DjwR1pWbkloTChdv+/KuavnzA5LiBJHkrEQBSQ5GY4fB6UgMBDKljU7otw52jmy7Ill+JbyJehCEA/MfIDzMefNDkuIEkWSsRAFZNIkY4CPqVPNjuTOqnhUYc+/99CkQhOiEqN4c8ubZockRIkiyViIAnD8OPz3v5CaCvXqmR1N3lRwq8CKJ1bgZOfEwsMLWXh4odkhCVFiSDIWogA0aWK8duwIbduaG8vdqFGmBpM6TgLg6R+eZvu57SZHJETJIMlYiHx26BCkpBjrs2ebG8u9eK7Fc4wJGENqeiq9Fvdi34V9ZockRLEnyViIfJSeDqNGGeu1akGVKubGc6++7vo1vWv3JiY5hsB5gZyJPmN2SEIUa5KMhchH334Lu3dDpUqwd6/Z0dw7WxtblvZbSmDVQBJSE+i3vB+JqYlmhyVEsSXJWIh81Lcv/OtfRp/ioj4ds6OdI2sGrKG6Z3X2X9zPsDXDSEtPMzssIYolScZC5CMvL5g1C/r1MzuS/OHp7MnqAatxsnNi+bHldFvUjYTUBLPDEqLYkWQsRD6IjQWLxewoCkaj8o1Y1X8VXs5ebPp7E/2W9SMlLcXssIQoViQZC5EPPvkE6taF7cW0J1BX/65sG74ND0cPfgr5iTHrxpgdkhDFiiRjIe5TWBhMmQIhxXwWwvre9Vk/aD22ypY5B+fw/eHvzQ5JiGJDkrEQ9+mVVyA+Hh57zBiDujh7qMpDTO40GYBn1j5DaEyoyREJUTxIMhbiPqxbB0uXGutffGFuLIXlhVYv0Lt2b5LTknlyxZMkW5LNDkmIIk+SsRD3KDERnn3WWP+//4OqVc2NpzB93fVrKrhVYGfYTkatG4XW2uyQhCjSJBkLcY8++QRCQ42GW5MmmR1N4fL18GXNgDU42Dow/9B8FhxeYHZIQhRpeUrGSqkuSqlgpVSIUur1W5R5VCl1UCn1l1JqW/6GKYT1efhhozY8fTrY25sdTeFr6dOSb7p+A8CodaPYf3G/yREJUXTdMRkrpWyBqUBXoB4wUClV74YypYFpQC+tdX3giQKIVQir0qEDnDxZ/Btt3c6/HvgXwxoPI8mSxMgfR8qAIELco7zUjFsAIVrr01rrFGAJ0PuGMoOAVVrrUACtdUT+himE9Zg3L2vcaQcHU0MxnVKKLzp/QUW3iuy7uI9XNr1idkhCFEl5ScaVgfPZ3odd35ZdLcBTKfWbUmqfUmpobgdSSo1USgUppYIuX758bxELYaL9+2HECPjgA5A2SwZPZ0/WPLkGOxs7/hf0P/aGF+EZMoQwSV6Sscpl241fQ3ZAM6A70Bn4r1Kq1k0f0nqm1jpAax1Qrly5uw5WCLP95z/G68mToHL7P6OEalG5BaOaGXNH9ljcg8OXDpsckRBFS16ScRjgm+29D3AhlzI/a63jtdaRwHagcf6EKIR1WLMGduww1nfvNjcWa/RZp8/oVKMTEfERtJvfjgtxN35NCCFuJS/JeC/gr5TyU0o5AE8Ca28o8wPwsFLKTinlArQEjudvqEKYJzERnnzSWH/++aI/PWJBcLRz5Icnf6B5peZcSbxCxwUdpUGXEHl0x2SstbYAzwEbMRLsMq31X0qp0Uqp0dfLHAd+Bg4De4DZWuujBRe2EIVr4kRIToYGDeCzz8yOxno52Tmxsv9KqnhU4djlYwxeNZh0nW52WEJYPWXWyDkBAQE6KCjIlHMLcTcOHoSmTY3133+HNm3MjacoOBpxlICZASSnJTO542Rebv2y2SEJYRWUUvu01gE3bpcRuIS4gypVjIZbzzwjiTivGng34Pu+xqxOb/z6BnvC95gckRDWTZKxEHdQpowx3OWUKWZHUrQ8Xu9xxgSMITU9lWFrhhGfEm92SEJYLUnGQtxCWprxnBiMwT1cXMyNpyia3GkyNTxrcCLyBANXDpQJJYS4BUnGQtzCggVQrx5s2GB2JEWXi70LK/uvxNXelR9P/sis/bPMDkkIqyTJWIhcXLxojLR1+jRERZkdTdHWuEJjZvSYAcDzPz0vE0oIkQtJxkLk4r33jNcWLWDQIHNjKQ4GNRzEiCYjSElLYfCqwSSmJpodkhBWRZKxEDc4cgRmGBU5PvoIbOT/kvumlGJK1ymZz48//P1Ds0MSwqrI14wQ2WidNf70s89C+/bmxlOcuDm4Ma/PPBSKD3//kDkH5pgdkhBWQ5KxENksXw6//QZeXvD++2ZHU/y0qdKGSR0nAfCfn/7DqSunTI5ICOsgyViIbEqXhurV4cMPjf7FIv+93PplHq/7OPGp8QxYMYC09DSzQxLCdJKMhcimUyf46y/417/MjqR4m9VzFhXdKnLgnwO8teUts8MRwnSSjIUga3APACcnsLU1L5aSwNPZk+k9pmOjbJi4YyJrg2+cCE6IkkWSsSjxLBZ48EEYOxbi4syOpuToVbsXn3b4FICnf3iaiPgIkyMSwjySjEWJ99VXcOAA/PSTdGMqbC8++CJtq7XlSuIV3v3tXbPDEcI08tUjSrRz5+Dtt431adPA1dXceEoaG2XDlK5TsLOxY3rQdJndSZRYkoxFiTZiBCQkQP/+0LWr2dGUTA28G/Bc8+fQaHos6sHZq2fNDkmIQifJWJRYy5bB1q3GjExffml2NCXbB+0+4JGqj3A54TJt5rQhODLY7JCEKFSSjEWJlJgIL75orI8ZAxUrmhtPSefq4MrSfktpUbkF4XHh9F3Wl2RL8p0/KEQxIclYlEgWC/TsCf7+MGmS2dEIgPJu5dkydAvVPatz7PIxntvwnMx/LEoMScaiRHJ3h+nT4dAhsLc3OxqRwdXBlWX9luFk58TsA7OZsnuK2SEJUSgkGYsSJSkpZ19iZ2fzYhG5a1apGbN6zgLgpU0vsStsl8kRCVHwJBmLEuXzz6FJE9i71+xIxO081egpXmr1Euk6nSdXPMk/1/4xOyQhCpQkY1FinDoFEybA6dMQG2t2NOJOJrSbQEClAM7FnKP3kt5cTbpqdkhCFBhJxqJE0Bqeeca4TT1kiMxTXBS42LuwesBqqnpUZU/4HoatGSYNukSxJclYlAirVsHvv4O3t/QpLkp8Svmweehm3BzcWBu8lml7p5kdkhAFQpKxKPaioqBfP2P9nXdknuKipmaZmpkNul7Y+AJ/nv/T5IiEyH+SjEWx99xzWeujRpkXh7h3TzZ4khdavoAl3UKnBZ3Ydnab2SEJka8kGYtiTWvo3h1atIATJ2Se4qLs046f0rduX+JT4+mwoAMLDy80OyQh8o0kY1GsKQWDB8Pu3VC7ttnRiPthb2vP0n5L6VevH5Z0C0NWD2Hp0aVmhyVEvpBkLIql1FR46SVjhC1RfNjZ2LH8ieW89+h7AIxaN4qQqBCToxLi/kkyFsXS8OHwxRewZInZkYiC8N/A/9K1ZldikmPotKAT0YnRZockxH2RZCyKnaAgWLTIWK9b19xYRMFQSrG031KaVGjCmatneHzZ4zLLkyjSJBmLYiU1FZ5/3ljv3BmGDjU3HlFw3B3dWdl/JRXcKrD17Fae3fCsDAoiiixJxqJYefNN2HV9XoEFC8yNRRS86p7Vmdt7LgDfHviW2ftnmxyREPdGkrEoNjZuhMmTje5Lf/wB5cqZHZEoDF1qdmFaN2NkrrEbxrLlzBaTIxLi7uUpGSuluiilgpVSIUqp129TrrlSKk0p1S//QhTizrSGiRON9ffeg4ceMjceUbjGNB/DS61ewpJuodfiXgRHBpsdkhB35Y7JWCllC0wFugL1gIFKqXq3KDcR2JjfQQpxJ0rBjz/Cp5/C67f8c1EUZ5M6TeKJek8QnxrPwJUDpUGXKFLyUjNuAYRorU9rrVOAJUDvXMo9D6wEIvIxPiHyzNUVXn1VRtkqqWyUDbN7zcavtB8H/jnA02ufJi09zeywhMiTvCTjysD5bO/Drm/LpJSqDDwGTL/dgZRSI5VSQUqpoMuXL99trELcZNs2ePppuHbN7EiENSjlWIrv+36PnY0di44s4tVfXiVdp5sdlhB3lJdkrHLZdmP/gS+BcVrr2/4ZqrWeqbUO0FoHlJPWNeI+Xb4MgwbB3LkwY4bZ0Qhr8aDvg6zsvxJbZcsXu75gwIoBJKYmmh2WELeVl2QcBvhme+8DXLihTACwRCl1FugHTFNK9cmXCIXIRXq60Yf4wgVo0wb+7//MjkhYk161e7F+0Hpc7V1ZcWwF3Rd1l2fIwqrlJRnvBfyVUn5KKQfgSWBt9gJaaz+tdTWtdTVgBTBWa70m36MV4rrJk+Hnn8HLCxYvBjs7syMS1qZzzc78OvRXvJy92Hp2K6PXjzY7JCFu6Y7JWGttAZ7DaCV9HFimtf5LKTVaKSW/3aLQ/fmnMbgHwPz54ONjbjzCerX0acmmIZtwsnNi3sF5vPvbu2aHJESulFnDxwUEBOigoCBTzi2KrqgoaNoUQkPh5ZeNGrIQd7Lg0AKG/zCcdJ3OqGajmNZ9GjZKxjwShU8ptU9rHXDjdvltFEWKoyO0bQstW8JHH5kdjSgqhjQewvw+87FRNszYN4PxW8abHZIQOUgyFkWKqyvMmwebN4ODg9nRiKLkqUZPsfbJtdjZ2PHxHx+z8thKs0MSIpMkY1EkrFgBv/xiDHsJ4OZmbjyiaOpeqzsftTNuqQxdM5SjEUdNjkgIgyRjYfXWroUnnoAhQyAhwexoRFH3SutXeKzOYySkJtD1+67Ep8SbHZIQkoyFddu6Ffr3N9Y7dDBuUwtxP5RSzO09l7pl6xIWG8as/bPMDkkIScbCeu3aBX36QHIyjB4t8xOL/OPh5MG4h8YBMG7zODaf3mxyRKKkk2QsrFJwMHTvDrGxMGAATJ1qzMwkRH4Z1mQYYwPGkpKWQqcFnSQhC1NJMhZWJy4OevUy+hT37AkLF4KN/KaKAvB1t68Z0WQEGs3wNcO5knDF7JBECSVfccLquLoat6cbNoRFi2SoS1FwbJQNs3rOomXlloTHhdN/RX+SLElmhyVKIEnGwurY2MDEibBjh3RhEgXP1saWZU8sw9vVmy1ntvDM2mdk2kVR6CQZC6ugNbzzDoSFZW1zdzcvHlGyVPGowqanNuFi78KiI4sYs26MJGRRqCQZC9NpDa+9Bu+/D126QNptZ8UWomA0rtCYNQPW4GTnxMz9MyUhi0IlyViYSuusCR/s7OCTT8DW1uyoREnVsUZH1j65NjMh91rcizPRZ8wOS5QAkoyFqT79FL74whhneuVK6NHD7IhESdexRkdWD1iNi70L60+t58FvH+R8zHmzwxLFnCRjYQqtoXlzeP114/2CBUZ3JiGsQZeaXTjx7AkCqwZyKf4SPRf3lG5PokBJMham+PNPyJjOeubMrCEvhbAWvh6+rB6wGv8y/hy6dIg+S/vIM2RRYCQZC1MEBMCjj8L06fDvf5sdjRC5K+Nchq3DtlLWpSx/hP7B8DXD0RlThwmRjyQZi0ITFAS7dxvrjo7GJBCjRpkbkxB3UrlUZZb2W4qjrSMLDi/grS1vmR2SKIYkGYtCsXQpPPww9O4NFy6YHY0Qd6edXztWD1iNjbJh4o6JHPrnkNkhiWJGkrEoUOnp8N//wpNPQlKSMdZ02bJmRyXE3evq35XRzUaTrtPpubgnF+Lkr0qRfyQZiwKTmGg0zPrgA2OIy6++MhprOTiYHZkQ92Zyp8k86PMg52PP031Rd6ITo80OSRQTkoxFgTh6FFq0MPoOe3jATz/Bf/4j0yCKos3Z3pm1A9dSs0xNDv5zkNZzWhMWG3bnDwpxB5KMRYGIiIC//oJateCPP6BTJ7MjEiJ/lHUpy9ZhW/Ev48+JyBN0WtCJs1fPmh2WKOIkGYt8k5yctd6unVEr3r8fGjQwLyYhCoJPKR+2Dd9GtdLVOB55nIb/a8gvf/9idliiCJNkLPLFnj1Qty78+mvWtsceM+YmFqI4quhekZ3P7OThKg9zLeUavZb0YvXx1WaHJYooScbivqSlwccfw0MPwZkzMGWK2REJUXgquFXgt+G/MSZgDEmWJPqv6M+BiwfMDksUQZKMxT0LD4eOHeHNN8FigRdfhGXLzI5KiMJlo2yY2m0q/2r6LyzpFnov6U1IVIjZYYkiRpKxuGtaw3ffQb16xiha3t5Ga+nPPzdG1hKipFFKMaXrlMxuT+2/a8+JyBNmhyWKEEnG4q5duwZvvAGxsdC9Oxw+DF26mB2VEOZytndm05BNtKzcktCYUB6a8xC/nv71zh8UAknGIo9SUiAhwVh3d4e5c2HOHPjxRyhf3tzYhLAWbg5ubB66mW7+3YhKjKLH4h7su7DP7LBEESDJWNzRr79Co0bw9ttZ2zp1ghEjZBAPIW7k5uDGjwN/ZHDDwSRZkui4oCP7L+43Oyxh5SQZi1sKDobHH4cOHYz1n382ashCiNuzUTbM6T2H3rV7E50UTbv57dj09yazwxJWTJKxuEl4uDHHcL16sGoVODvDhAmwb5+MKy1EXjnYOrDsiWV09+9OTHIMfZf2ZUfoDrPDElZKkrHI4dw5qFkTZs82JncYORJCQmD8eGkpLcTdcrB1YPWA1QxsMJD41Hgenvsw3x/+3uywhBXKUzJWSnVRSgUrpUKUUq/nsn+wUurw9eVPpVTj/A9VFJT4+Kz1qlWNFtJPPGFM9jBjBlSqZF5sQhR19rb2fPfYd4xuNhqNZtiaYXzyxydorc0OTViROyZjpZQtMBXoCtQDBiql6t1Q7AzwiNa6ETABmJnfgYr8d+ECvPaakWx37szavmSJMXhH7drmxSZEcWJnY8f/evyPtx5+izSdxhu/vsGAFQO4mnTV7NCElchLzbgFEKK1Pq21TgGWAL2zF9Ba/6m1zpjYcxfgk79hivx04gQ88wxUqwaTJhn9hVeuzNpvZ2daaEIUax+0+4Cl/Zbi7uDO8mPLaTGrhcyJLIC8JePKwPls78Oub7uVZ4CfctuhlBqplApSSgVdvnw571GKfPHbb9CtmzGhw5w5xhCW/foZkzxMnmx2dEKUDP3r9ydoZBD1y9XnVNQpHl/2OLHJsWaHJUyWl2ScW0/SXB92KKXaYiTjcbnt11rP1FoHaK0DypUrl/coRb747Tdj2EpnZ6NhVnAwLF8OzZubHZkQJUstr1r8OPBHyjiXYevZrQTODeSfa/+YHZYwUV6ScRjgm+29D3DhxkJKqUbAbKC31vpK/oQn7kVaGmzYYNR6v/oqa/vQocYMS6GhRsMsf3/zYhSipPPz9GP3v3bjX8afQ5cOETg3kHNXz5kdljCJulOLPqWUHXASaA+EA3uBQVrrv7KVqQJsAYZqrf/My4kDAgJ0UFDQvcYtbqA1HDoEq1fDvHlGwgWoX99oFS2EsE6Xrl0icF4gJ6+cxM3BjendpzO40WCzwxIFRCm1T2sdcOP2O9aMtdYW4DlgI3AcWKa1/kspNVopNfp6sbcBL2CaUuqgUkqybCFavBhq1ICmTeH9941EXKMGfPQRbJJBf4SwauXdyrNt+DbaVmvLtZRrDFk9hFXHV5kdlihkd6wZFxSpGd+bpCRjrGhv76xnvatXQ9++xrbevY0+wu3bG4N2CCGKhrT0NMZtHsdnOz/D2c6Zpf2W0rN2T7PDEvnsnmvGwnxRUbBoEfj4QKlS0KMHfPll1v7OneGPP4x+wzNnQseOkoiFKGpsbWz5tOOnPN3kaRItifRa0ouH5z7MkUtHzA5NFAL5yrZiCxYYz3y9vGDwYGPM6NRUaNwYmjXLKufiAg89BLa25sUqhLh/NsqG2b1m80n7T3BzcOOP0D9oObsl//npP9LaupiTZGwF0tONRlYzZ8KxY1nbw8ON93Z20KQJvPoqnD4NBw/CSy+ZF68QouAopRjXZhxhL4YxsMFAEi2JfL3naxr+ryErj6288wFEkSTPjE1w8SIEBRmDbezdC7t2QUyMse/99+G//zXWz52DzZuNZ8ClSpkXrxDCPHvC9/DqL6+y/dx2AIY3Gc707tNxtJOZW4qiWz0zlmRcwKKi4PBhePTRrG3Vq8OZMznL+fpCq1bG7ejevRFCiEypaalM2zuN1za/RkpaCjU8a/Duo+8yuOFglMptXCZhrSQZF7CUFGPM5yNHjOXwYeM1LMzYHxEBGYOO/fvf8PffRmvogAAjCfv63vrYQggBsPP8TgasGMD5WGOE4sfqPMbUblOp6F7R5MhEXkkyzicpKcZz2xMnoEwZCAw0tm/ZYnQnupGTk9H/99tvjTGhhRDifiSkJvDpjk+Z9OckElIT8HTyZGHfhXTz72Z2aCIPJBnfo2XLjG5DJ0/CqVNw9qzR4AqM4SaXLzfWL182WjQ3bJhzqVFDWjkLIfLf2atnGb1uNBv/3ghAj1o9eOXBV3ik2iMmRyZuR5LxDeLijAZSZ88ar2fOGMn21Cn45ReofH1eqv79sxIuGP13q1aFWrWgQwd45RVTwhdCCNJ1Op/88Qnv/PYOlnQLAJ1rdOa7x77D29Xb5OhEbkpUMk5LM57Rhocbz2xDQ40EmtEwau9eaNHi1p/fsgXatjXW16yBkBBjUoVatYzGV47SiFEIYUXCYsOYc2AOn+38jNjkWHxL+TKl6xR61e6FjZIerNakWCTjuDijW9A//8ClS8b6yJHGc1kw1tetM/Zl3ErO0KePMWwkGJ+vVi3nUrUq1KxpJN06dbKOKYQQRcXZq2fpvLAzJ6+cBKChd0O+6PwF7avn0qBFmOJWydjOjGAyREcbz1ojI7NeIyONBk+dOhllduyAIUOMBJuQcPMxevQwaqsAsbFGggaj5XLlylCpUla3oQzlyxvHkiEjhRDFSbXS1Tg0+hCz9s1i4o6JHIk4QocFHWjl04qRD4xkaOOh2NpIIxZrZFrN2Nk5QCcl5V4zfvZZ+OYbY33PHmjZMuMzRnItX96YFKFSJXjjDWPMZjBuRytl7HdwKISLEEIIK5VsSeaj3z/iy91fEpscC0D9cvV5pfUrPNXoKexsTK2LlVhWd5vazS1A29gEUa4clC1Ljtc2bbKe7yYlwfnzUKECuLkZyVYIIUTexKfEs+yvZby55c3M8a2bVWzG+23fp0vNLvJMuZBZXTI2uzW1EEKUJDFJMSw5uoS3f3ubiPgIAAKrBrLk8SUyaEghkikUhRCiBPNw8mBUwChO/+c0EztMpKxLWbaf207tb2ozacckElMTzQ6xRJNkLIQQJYirgyuvPfQah0cfpmP1jsSlxPHa5teo9HklZgTNIC09zewQSyRJxkIIUQJVdK/IpiGbWPvkWppVbMbVpKuMXj+apjOasubEGlLTUs0OsUSRZCyEECVYz9o9CRoZxHd9vqOKRxWORBzhsaWPUfPrmvxw4gdJyoVEGnAJIYQAjO5Q04OmM3XvVE5FnQLA3cGddn7taO/XnjZV2tCofCPpq3wfpDW1EEKIPLGkW/hq11d8e+Bbjkcez7HPt5Qv/3rgX4xoMgJfD5n79W5JMhZCCHHXzl09x69nfmXr2a38dvY3wmLDMvcFVg1kdLPR9KvXD3tbexOjLDokGQshhLgv6TqdDac2MO/gPDac2kCixegOVdWjKqOajaK1b2sCKgXg6uBqcqTWS5KxEEKIfBOVGMWyv5bx5a4vCb4SnLndVtnSsHxDWlVuRUuflrT3ay+3s7ORZCyEECLfpaWn8UPwD2w+vZldYbs4fOkwaTpnX+VWPq0Y2mgoTzd9Gke7kj0HrSRjIYQQBS4+JZ59F/exK2wX289tZ8uZLZm3s8u7lqdnrZ608mlFs0rNaOjdsMS1zJZkLIQQotDFp8Sz9K+lfPzHx4REheTY5+XsRY9aPRjRZAQPV324RExaIclYCCGEadJ1OkEXgvgj9A92h+9md9huzsWcy9xf3bM6I5qMYFjjYcX6GbMkYyGEEFZDa03wlWAWHl7I/EPzc3SZquhWkcYVGtO4fGN8S/nyQMUHaOXTClUM5tCVZCyEEMIqpaWnsfn0ZuYcnMPa4LUkWZJuKuPp5EntsrUp51IOLxcvGpdvzKPVHqVu2bpFqlGYJGMhhBBWLy09jdCYUP48/yeno09z5uoZ1p9anzkHc27qlK1DQKUAanvVprZXbWqUqUG9cvVwsnMqxMjz5lbJ2M6MYIQQQojc2NrY4ufph5+nX+Y2rTXnYs5xPuY8kQmRXLx2kR3nd/DDiR+IT43nROQJTkSeuOlYldwrUcWjCmWcy+Dj7kPHGh15yPchKrhVsLpb3lIzFkIIUWRdjr/MicgTBF8JJjgymO2h27kcf5nQmNCb+jtn8HD0oJZXLTycPAisEkjV0lWp4FaB8q7lKe9WnrIuZbGzKZi6qtymFkIIUWJY0i2ExoRyIe4C0YnR7L2wl9/O/sbRiKNEJ0Xf9rMKha+HL/XL1cevtB9eLl74lPKhvGt5KrhVwMvFC3cHd0o5lsLJzumuatlym1oIIUSJYWdjR3XP6lT3rA4Y8zaDccv7Uvwldp7fyYW4C/wd/Tf/XPuHS/GXuHTtEpfiL3El4QqhMaGExoTe8Twu9i74lPKhsntlPJ09M5N0xnLj+1vGm5eLUkp1Ab4CbIHZWutPbtivru/vBiQAw7XW+/NybCGEEKKwKKWo4FaBx+o+dssyqWmp/B39Nwf/OUhEfASRCZGExYZxKf4SF+IuEJccR2xyLLHJsSSkJnDyyklOXjl5X3HdMRkrpWyBqUBHIAzYq5Raq7U+lq1YV8D/+tIS+N/1VyGEEKJIsbe1p07ZOtQpW+eOZWOTYwmLDSM8Njwz1fDBAAAACgVJREFUQcelZCXrG99vZ3uux8lLzbgFEKK1Pg2glFoC9AayJ+PewHfaeAC9SylVWilVUWt9MQ/HF0IIIYqkUo6lqFeuHvXK1ctTeTUi9+fLeUnGlYHz2d6HcXOtN7cylYEcyVgpNRIYef3tNaVUMOYpC0SaeH6zyfWX3OsvydcOcv1y/eZef9XcNuYlGeeWxm9sgp2XMmitZwIz83DOAqeUCsqtRVtJIddfcq+/JF87yPXL9Vvn9edliowwIPuo3T7AhXsoI4QQQohc5CUZ7wX8lVJ+SikH4Elg7Q1l1gJDlaEVECPPi4UQQoi8ueNtaq21RSn1HLARo2vTHK31X0qp0df3Twc2YHRrCsHo2jSi4ELON1Zxu9xEcv0lV0m+dpDrl+u3QqaNwCWEEEIIQ15uUwshhBCiAEkyFkIIIUxW5JOxUup5pVSwUuovpdSn2ba/oZQKub6vc7btzZRSR67vm3J9KE+UUo5KqaXXt+9WSlXL9plhSqlT15dh2bb7XS976vpnHQrnqnNSSr2ilNJKqbLZthXr61dKTVJKnVBKHVZKrVZKlc62r1hf+/1QSnW5/nMJUUq9bnY8d0Mp5auU2qqUOv7/7Z1/jFxVFcc/33RtG4HS3WLpCijsHzSAxrYBdaXFFQjBTW1i1LgkJEpjlB8h/kgsYP1RQUjaRlKMia0haYVdCGUBG5tqSwnlh2Ch1lZN6EIXFilQEEpVaKhAj3/cM5m305ltx2Xn7cycT/Ly7j333jf33Pdmz7xz757r3/dvu7xN0v1+L+6X1JppM+bPQi2RNEHSXySt93wz6T5VUr9/75+S1NlQ+ptZ3R7A54DNwCTPT/fzmcBOYBJwGjAITPCyJ4BO0v9G/x74vMuvBFZ6uge4y9NtwLN+bvV0q5etBXo8vRK4IocxOIW0uO554IRm0R+4CGjx9FJgabPoPooxm+Dj0QFM9HE6M+9+VdH/dmCOp48Dnvb7vQy41uXX1vpZqPEYfA+4A1jv+WbS/TfANzw9EZjaSPrn/gUb5c1ZC1xYRn4dcF0mv9EHvx3YlZFfAqzK1vF0CylCi7J1vGyVy+R1CgahE9iYwxj0A58Ahiga46bR3z/7i0BfM+pe5TgN62fpWNXbAawjxcwfANpd1g4M1OpZqLG+JwMPAOdTNMbNovsU4Dl80XFG3jD617ub+nRgnrsUHpJ0jssrhec8ydOl8mFtzOxd4F/AtBGuNQ3Y73VLr1UTJC0AXjSznSVFTaF/hoWkX7jQfLpXQyV96g53Ic4GtgInmsc18PN0r1aLZ6GWrAAWAYcysmbRvQP4J7Da3fS3SjqGBtJ/3O9nLGkzMKNM0WJS/1uBTwPnAGsldVA5POdIYTurbXNUIUBHyxH0/wHJXXtYszKyutN/JN3NbJ3XWQy8C/QVmlXoV13pPkbUa7+HIelY4B7gO2b2b1Xe2L0Wz0JNkDQfeNXM/iyp62ialJHVpe5OCzAHuNrMtkq6heSWrkTd6T/ujbGZXVipTNIVwL2WfAdPSDpECgJeKTznHk+Xysm02SOpBTge2OfyrpI2W0gujKmSWvxX1JiEAK2kv6SPk+ZCdvofo5OB7ZI+SYPoP9K9h7SoApgPXODPADSI7mNE3YetlfQBkiHuM7N7XfyKfJc4Se3Aqy6vxbNQK84FFkjqBiYDUyT10hy6F/q2x8y2er6fZIwbR/9a+v3HYB7hcuB6T59OciUIOIvhk/fPUpy8f5L0Jl2YvO92+VUMn7xf6+k20lxFqx/PAW1edjfDF/FcmeNYDFGcM254/YGLSdt4fqhE3vC6j2LMWnw8TqO4gOusvPtVRf8F3AasKJEvZ/ginmW1fBZyGIcuinPGTaM78Agw09NLXPeG0T/3L9gob85EoBf4O7AdOD9Ttpi0gm4AXy3n8rO9/iDwS4pRyCaT/sDuJq2268i0Wejy3cBlGXmH193tbSflOBZDuDFuBv39M18Advixsll0H+W4dZNWIQ+S3P2596mKvs8luQf/mrnv3aR5vQeAZ/zclmkz5s9CDuPQRdEYN43uwCxgm9//35IMY8PoH+EwgyAIgiBn6n01dRAEQRDUPWGMgyAIgiBnwhgHQRAEQc6EMQ6CIAiCnAljHARBEAQ5E8Y4CMYApV20jnQMed01kvYc4ZI1QdKQB5N4P6+35ijqrSmMRxA0I+M+AlcQ1CmdJfn7SEEIlmRkB2vWmyAIxjVhjINgDDCzP2Xzkg4Cr5XKR4ukSWYWRj0I6pxwUwfBOEHSbEmPSDrgm5hfXlL+dXdvnyfpbkn7SbsWIanFN1PfJemgpJck/VzS5Ez7Fkk3SBqU9Lak1yQ9Kmlumb70+Abub0naVqHOpZJ2Zq51u8cHPpKeF0ja7u0GJX3r/xqwIGgg4s04CMYHU0ibxq8ArgcuA34lacDMHiyp2wfcCXyZ4ne4F/gCsBR4DDgDuAE4FfiS17kG+C4pTOAO/8yzSbF3s8wDZgI/At7266yXdKqZ7QeQ9E3Svq53kfaO/TBwE/ApSXPM7M1ySko6A9hACmvYQ4odvAQ4FnjviKMUBA1KGOMgGB8cR9ps4kEASQ+Ttse8BCg1xv1mtqiQkTQP+CrwNTO7zcWbJe0DeiXNMrMdpHnsTWZ2S+ZavyvTlynALDN7w6+/lxRcvxu4Q9IEkoHeYmY9mX7sIgXzXwj8ooKePwT+A1xkZm95u8dIcYLragepIHg/CTd1EIwPDmTfgH0e+BngI2Xq3leSvxj4L3CPu6JbfAu4TV5+np+fBLol3ShprqSJFfryeMEQO3/zc6EvM0mbuPdlG5nZo8DzwGcrKUn6QbChYIi93QvAH0doEwQNTxjjIBgfvFFGdpC0k0wpL5fkp5N2MHsTeCdzFPZ2nebnm4CfAAtIb7CvS1ot6YSS6+3LZjILxAp9Kbi1S/sBsJfD3d5Z2oFXysjLyYKgaQg3dRDUH6Vbrb1OmtudV6H+SwBm9g5pTnmppBnAfOBm4IMkN/fRUjDWM8qUzSDNB1fiZeDEMvJysiBoGuLNOAjqnz+Q3lqPN7NtZY7D5mLNbK+Z3QpsBj5W5ecNkN5ke7JCSZ8BPgo8NELbx0mu8mMy7U4Bzq2yD0HQUMSbcRDUOWa2RdKdQL+km0kbox8iraTuBq4xs6clrSMFHtlOcovPJs03r6ry896T9GNglUfr6gVOAm4kzXOvHqH5z4CvAJskLSe5139KuKmDJieMcRA0BpcCV5NWMi8mzTcPARspGrqHSYbwKpJr+h/AMpIRrQoz+7WkA8D3gXWk+eoNwKJK/9bk7Z6S1A0sJ/1b1Isk13kn0FVtP4KgUZBZ6fRTEARBEAS1JOaMgyAIgiBnwhgHQRAEQc6EMQ6CIAiCnAljHARBEAQ5E8Y4CIIgCHImjHEQBEEQ5EwY4yAIgiDImTDGQRAEQZAz/wNXsLSletxQBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib 绘制精度和召回相对于阀值的函数图\n",
    "def plot_precision_recall_vs_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label=\"Precision\", linewidth=2)\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label=\"Recall\", linewidth=2)\n",
    "    plt.xlabel(\"Threshold\", fontsize=16)\n",
    "    plt.legend(loc=\"upper left\", fontsize=16)\n",
    "    plt.ylim([0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_precision_recall_vs_threshold(precisions, recalls, thresholds)\n",
    "plt.xlim([-700000, 700000])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF5CAYAAACV7fNGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd5RV5b3/8c+XoTNUIYBiw2u9GixjQQ1ijy5sXFGj0ViJV40Xf/FGo0S9UX4qejGuRL1iiSVqAjExxlxLDPZgGTDyEwuWgDgRg1QRKQPf3x/PGffMOOXMzDn7OeX9Wuus/Zx9nnP2l+0sP7s+29xdAACg9HWKXQAAAEgHoQ8AQJkg9AEAKBOEPgAAZYLQBwCgTBD6AACUCUIfAIAykXrom9lgM3uhlT53mdlMM5uYVl0AAJS6VEPfzPpLuldSrxb6jJVU4e4jJQ03s23Tqg8AgFKW9p7+BkknSlrZQp/RkqZl2k9J2j/PNQEAUBY6p7kwd18pSWbWUrdekmoy7aWSdm/cwczGSxovSZ0799+jT5/hX322dGmY7rST1KNH4+VLa9dKa9ZIX34prV8vrVsn1daGdm2tNHSoNGRI6L9xYzK/blpZKXXv3o5/PAAAOTJr1qzP3H1QW7+XauhnaZWkuriuVBNHI9x9qqSpklRVVeXV1dVffTZihDRnjvTgg6H9zjvS9OnSjBnSzJkh9FtSUyP16hWmX3zRdJ8zz5Q+/TS8Vq2SrrtO6tdPWrxY2nNPacst6+qUVq8OGyKNX8uWhenq1dJ3vysNGiRtvXUb1xQAoCyZ2YL2fK8QQ3+WwiH9lyWNkPRue37kpZekCy+Unn++4fytt5Z23DG8ttoq2bNfvlwaMyb0mTcvTLt2lQYPDq/u3aUXXwzz77674W8ee2zD93vtJS1aFDYKWtvIkKSf/zxM+/WT7rtPGj5c+td/bdM/FwCAVkUNfTPbSdLJ7l7/Kv1HJL1gZptKOkLSPm35zdraMD3//DCtrJTGjQuBfsAB0iabNP/dZ58Ne+5bbSVttpnUt69U/0zEb38rvf12siHwt79JkydLm24q9ewZjjBI0quvJt/p0UMaMCB59e+ftOfOlR5/POm7fLl09NGh3b+/dPLJ0qhRob14sfTPf4bp4sXS/Plho2b06FBzba00bFj47saNUqd2Xq2xfn04CrF8efi9nj3Db3/+ubRyZXjVbzf1/t13w/peuzYcLVm1KpmuXBk2iHbdVTr0UOmb3wyf1dZK224bTrvU9e/TJ6zbfv3a928BADRkhfho3cxV/odKet7dF7XUt/Hh/fohPXGi9KMfSb1756nQRl5/PYTxkCHhNXhwCM3W1NaGEOzZU3rttY7V0KdPCFZJ2n13afZs6fTTQ4guWZK8amrC6Yf99w8BXxf0jU9pdOuW3dGKfOvRI2wQSNI++0iHHBJqrb8x8eabYWOvW7dwrcaZZ4YNJgAoNWY2y92r2vy9Qgz9tmgc+ueeK/3ud9JvfiMdeGDEwtqptjZsqCxZEg7177xzON//jW8k01dekf70p9DfLIR3rlRUhD3rJUuSeWZhY6JPn7ABVddu/L53b+n990MQDxwYjrL06tVw+tJL4UjFQw9Jm28ejqbMmxdCepNNkr5vvZW7f9PQodInn4QNgrPPDuvx88/Da9WqMF2yRNphh3CUZ9WqcKSkf//wbxk0iIs3ARQWQr+e2lqpcyFerZBja9eGf2enTtJ774V5AweGixdXrAjXFHz4YTjisMkm4TVgQJjOnRsO5ffvH0K+f//wqqwMIb9hQzid0KdPOALR8g0X+fP55+Fix549w4bcyy+Henv3brhB8bvfhQsjt946rIu//jX3tey5Z1inu+0WNgxmzw4bYUceGepctkyaNUuaNCkcudlpp9zXAAASoR+7DBQY93BtwfLl0qOPStdeG+Z36hQ2gnr3DhsMvXuH0wZ112EMHhyCXQobF8uXd6yOTTcN13306hU2ooYMkf7lX8LGQs+eUpcu4VV3p0dtbXI6qr3XZQAofYQ+kAcbNkh/+Us4/VB3EWfv3uEoyR//KG2zTXLU4eqrw+mAFSvavpxOncIFmE055phw8WOvXuF0z+abhyMLs2eHiyF79AgbEZ9/Ln38cbhldN26cFRnyZLw2wcdFDZAzOIdtQGQO4Q+UEDmzAmH/Wtqwh0Wn34agnvZstiVJTbfPGyk3HSTtN9+YQNnwICwcdF4YCsAhaW9oV8GZ76B9H3zm2GvuyW1tWGPfN26cKqhc+cwzyxcG/DWW+HagXffDacG3ntP2mOPcGRhzpwwf8mSMC5Enz7S3/8eAnv2bGnkyDDOxNNPN7/8hQvD9Hvfa77PaaeFWnr2DGNHuEsHHyx95zvlcd0MUGrY0wfKwOrV4dqBpUvDRkFNjXTDDclAVB2x997hAso99ggbBXvvHTY+Bg4MYz1UVHR8GQAa4vA+gJxYvly68soQ5KtXh6MKH38cAr29d0Xsvbd02GHheoM1a8JryZJw1GDUqLBBAiB7hD6AVLiH0w7z5oVnWrz+ejJCZW1tGK56s83C0YS26NIl3A75rW9Jhx8eLpb88suw4bF2rTR2bLjmAAChH7sMAE349FPp97+Xbr89jFvQvXt4zZ4tVVcnw2a3Rb9+YeCtYcOk446TdtklnEoAygmhD6AorV8fLlp85BHpnnvCMMpDhoQ7CHr2DGMotHZRpFkYjfM//kPabjtuS0TpI/QBlCx36Y03pI8+Cq+f/CQc6v/ww+a/8/DD4eLCLbZgIwClh9AHUJb+/OdwJ8Kf/9x8n+OOC7cxrl0bRkQ87DCuD0BxI/QBlL01a8KFgGvWhCMCdU+cbMo220jTpoWLBzkSgGLD4DwAyl737g0fT71woXTHHWFo5BkzwoWFixeHzz74IBz+rzN6dHgc83e/y0YAShd7+gDKypo10rHHSk8+2Xyf006TJkwIzzpgDAEUovbu6fMcLwBlpXt36YknwsWB7uEpjP/+7w373HeftPvuYShjM+mII8J4BECxY08fADI++CBc6Ne9ezgi0JSBA8Pe/2GHSSefHEYU7N493ToB9vQBoIO22Sbs/X/5ZRg46KWXpEsvbdjns8+kTz6R7r03jBzYo4f0pz/FqRdoK0IfAJpQUSHtu6907bVhQ2DRojC08AMPhGGH6xszJpwGMAsXA95yS7igcP36OLUDzeHwPgC004IF0lZbtd7v/POlK66QvvGNvJeEMsHhfQBI2ZZbhqMAn3wiXXNNmLf11l/vd8st4eiAWThlAMRC6ANABw0ZIl1+edgA+PDD5M6Axx77+sh/+++fnAo47zzp178OIwUCaeDwPgDk2WefSbvu2vLjhhcv5mmByB6H9wGgQA0cGJ4U6C4tXy5deGF4PHB9gwaFvf/f/CZOjSgPhD4ApKhvX+nmm8OwwO7SpEkNPz/ppBD+J56YDBkM5AqhDwARXXaZtHFjeEZAfdOmhav9zaR58+LUhtJD6ANAZGbS2WeHPf/nnpP69Wv4+fbbhz7Tp8epD6WD0AeAAjJqlLRsWRjYZ/z4hp+dcII0bFjzQwQDrSH0AaAAde4s3X572Pt/4olkfk1NGPr3P/8zXm0oXoQ+ABS4ww//+r38N94YHv373ntxakJxIvQBoAh07Rr2+jdsSObNnSttt1043w9kg9AHgCLSqZO0erV05JEN55tJ69bFqQnFg9AHgCJT9zjfjRsbzu/WTXrwwTg1oTgQ+gBQpMwaHu6XpFNOkd5/P049KHyEPgAUsU6dwrn++k/v23Zb6cc/jlcTChehDwAlYN99wxP76lx3XTgScM450uefx6sLhYXQB4ASceKJ0sKFDefdeafUp490221xakJhIfQBoIQMGxYO9y9YIFVWJvPPOy/s+Rf509TRQYQ+AJSgLbYIh/WffLLh/E6dpN/+Nk5NiI/QB4ASdthhYe9+332TeePGSXvtFa8mxEPoA0AZeOkl6Ze/TN6/9hqH+8sRoQ8AZeL006UlSxrOO+CAKKUgEkIfAMrIgAFhJL/evcP7F17gUb3lhNAHgDJjJi1dmrzv0YOL+8oFoQ8AZahzZ2nixOT9uHHSPvuw11/qCH0AKFNXXy29+GLy/pVXwl5/4wf5oHQQ+gBQxvbbT6qtlc4+O5lXUSF9//tff5gPih+hDwBlrqJCuuMO6aqrknlTp4ZTAB9+GK0s5AGhDwCQJF15pfT22w3nbbONdOyxcepB7hH6AICv7LBDGLDnzjuTeX/4g3TKKQzkUwoIfQDA15x1VsMr+R98UJo8OV49yA1CHwDQpG7dwkV+dS69VLrgAm7rK2aEPgCgWRUVUnV18v6WW8Jtfd/73teH9EXhI/QBAC3aYw9p/XppwoRk3n33SQMHSmPGcK6/mBD6AIBWde4s3XST9MEHDef/6U/SRRfFqQltR+gDALI2fHjYs1+/XhoyJMy7+Wbpk0/i1oXspB76ZnaXmc00s4nNfN7fzP7XzKrN7Pa06wMAtK5zZ2nhwuT9ppsygl8xSDX0zWyspAp3HylpuJlt20S3UyU94O5VknqbWVWaNQIAstO5szR9esP3KGxp7+mPljQt035K0v5N9FkiaWcz6ydpc0kLm+gDACgAxx8v7bln8n7bpnblUDDSDv1ekmoy7aWSBjfR50VJW0q6UNLbmX4NmNn4zOH/6sWLF+erVgBAFl55JWm//75kxhX9hSrt0F8lqUemXdnM8q+UdK67/1TSO5LOaNzB3ae6e5W7Vw0aNChvxQIAWmfWcBAfSdp55zi1oGVph/4sJYf0R0ia30Sf/pJ2MbMKSXtLYnsRAApcRUXYux82LLx/662wMbBxY9y60FDaof+IpFPNbIqkEyTNNbNrGvW5VtJUSSskDZD0ULolAgDa6/33pb59k/ejRsWrBV+Xaui7+0qFi/lelnSgu7/h7hMb9XnV3f/V3Svd/VB3X5VmjQCA9uvWTVq+XKqsDO9fekk67TTO8ReK1O/Td/dl7j7N3RelvWwAQDpqapL2/fdLnTpJkybFqwcBI/IBAHKuTx/piy8azps4UTrqqDj1ICD0AQB50bNnOKx/993JvMcek66/Pl5N5Y7QBwDk1RlnSP/8Z/L+0ku/fhQA6SD0AQB5N2iQ9I9/JO8rK6XPPotXT7ki9AEAqRg6NDyetw5jq6WP0AcApGbChHB4v85mm3E7X5oIfQBAqq69NjyKVwqH/MeNi1tPOSH0AQCp+/vfk/bDD0vLlsWrpZwQ+gCA1HXt2jDoBwyIV0s5IfQBAFH069fwwr5vfzteLeWC0AcARDNhQtJ+8klp8uR4tZQDQh8AEFX9q/cvuUSqro5XS6kj9AEA0a1enbT33FNasCBeLaWM0AcARNejR3gMb52ttopWSkkj9AEABWHffaUHHkjem8WrpVQR+gCAgnHyyQ3fc2FfbhH6AICCsnFj0r7kEmnduni1lBpCHwBQUMwajtjXrRvj8+cKoQ8AKDhbbdVwsJ5OpFVOsBoBAAXp8cel/fZL3v/iF/FqKRWEPgCgYL3wgrT99qH9gx/EraUUEPoAgIJlJv3qV8n7RYvi1VIKCH0AQEGrqkraQ4fGq6MUEPoAgII3aVLSfuedeHUUO0IfAFDwLrssae+4o/TFF/FqKWaEPgCgKEyfnrSnTIlXRzEj9AEAReH448NevkTotxehDwAoGuedF6bLlzd8HC+yQ+gDAIrG+PFJ+5xz4tVRrAh9AEDR6NpVGjs2tB98MG4txYjQBwAUlRtuSNo//GG8OooRoQ8AKCrDhyftKVOkmpp4tRQbQh8AUHSWL0/aw4bFq6PYEPoAgKLTt690883J+2eeiVdLMSH0AQBF6cILk/ZBB0kffxyvlmJB6AMAitZrryXtE06IV0exIPQBAEWrqkq6+OLQnjkzbi3FgNAHABS1q69O2o8/Hq+OYkDoAwCKWvfuUo8eoX3kkZJ73HoKGaEPACh6TzyRtK+4Il4dhY7QBwAUvVGjpMGDQ/v+++PWUsgIfQBASbj++jBdsIBD/M0h9AEAJeE730na774br45CRugDAEpC165Ju/79+0gQ+gCAknHccWE6dWrcOgoVoQ8AKBm9e4dp375x6yhUhD4AoGQcc0yYvvde3DoKFaEPACgZm2wSpvPmSevXx62lEBH6AICSsf/+SXvs2Hh1FCpCHwBQMioqpL32Cu3HHpOWLYtbT6Eh9AEAJeUPf0jaAwbEq6MQEfoAgJIyZEjyuF1Jev31eLUUGkIfAFBybrghaZ9xRrw6Cg2hDwAoSZMmhekbb8Sto5AQ+gCAkjR+fNKeNy9eHYWE0AcAlKSBA5P2YYfFq6OQpB76ZnaXmc00s4mt9LvVzI5Kqy4AQOmpO5/P43aDVEPfzMZKqnD3kZKGm9m2zfT7lqQh7v7HNOsDAJSW669P2p04tp36nv5oSdMy7ack7d+4g5l1kXSHpPlmdkx6pQEASs2gQdLRRyfvy31M/rRDv5ekmkx7qaTBTfQ5TdJbkiZL2svMftC4g5mNN7NqM6tevHhx3ooFABS/+oP1PPxwvDoKQdqhv0pSj0y7spnl7yZpqrsvkvQrSQc27uDuU929yt2rBg0alLdiAQCl4ZxzwvQXv4hbR2xph/4sJYf0R0ia30Sf9yUNz7SrJC3If1kAgFJWt3/Yp0/cOmJLO/QfkXSqmU2RdIKkuWZ2TaM+d0k60Myel3SepBtTrhEAUGKOytwL9vbb0oYNcWuJqXOaC3P3lWY2WtKhkiZnDuG/0ajP55LGpVkXAKC07bJL0n74YemEE+LVElPqNzC4+zJ3n5YJfAAA8q5Xr6T96KPx6oiNuxYBAGXhggvC9IEHpFdfjVtLLIQ+AKAsXHJJ0t5773h1xEToAwDKwrBh0v33J++XLo1XSyyEPgCgbHz3u0m7/p5/uSD0AQBlZccdw/TOO6VVq+LWkjZCHwBQVqZNS9ozZsSrIwZCHwBQVnbeWRo1KrTHj49bS9oIfQBA2dk/MyD8p59K8+dHLSVVhD4AoOxcdVXSvv32aGWkjtAHAJSdLl2ko48O7euuk9zj1pMWQh8AUJZuuy1pdyqTNCyTfyYAAA1tuql09tnJ+9Wr49WSFkIfAFC27rgjaU+YEK+OtOQ89M2sR65/EwCAfKl77O5dd8WtIw2thr6ZdTWzb2XaFWZ2VCtfudrMfpqT6gAAyLN77w3TjRvj1pGGbPb0B0h6OtPuLOnXrfQfKqlfR4oCACAtI0Yk7VJ/5G42ob8285K7r5VUW/9DM3vAzPrWmzVU0pycVQgAQB7Vv3L/2mvj1ZGGbEJ/o6QNZnaHmX0uqdLMlpnZ52Z2qKTvSHrTzPbM9B8haWae6gUAIOcOOihMH3kkbh351pYL+W6WdIykLyQdK+lvme+vkPQTSU+Y2X9K+tLd5+a6UAAA8uWn9a5Ee+KJeHXkW+cs+hwgyd39TUkys1p3f87MPst87u5+T+b9I5Km5KlWAADyYr/9kvYRR5TuCH0t7umb2e8Ugjwbe2em3TpUEQAAEUyalLTXro1XRz61dnj/F5JGS5KZjTSzMyR1NbPTJG2e6dPZzO6UNE7SwZKONzPLU70AAOTFxRcn7dNPj1ZGXrUY+u4+Q9Ibkkwh/Cco7Mn/UFJPSWskVWY+38vdn5NUI2lU/koGACD3unaV9sxckv7r1m5OL1LZXsjn7n6tpN0krXb3Ee6+k8JV+qvd/Sx3X5np+6ykkbkvFQCA/HrooaRdW9t8v2LV1mF4u0uqP8yuSfpNoz5vStqjI0UBABDDNtsk7RUr4tWRL9mGfjczu0DSmZImmNnZZnaypH0lTTazynp9P5A0Pcd1AgCQiuHDw/Smm+LWkQ/mrdyXkAn05yStk1TXubOkXpnXEEldJM2T9JSku939jXwV3FhVVZVXV1entTgAQInbemtp/vzQLtRb98xslrtXtfV7rd6n7+6r1MrhejPbQtJBkk6SNNvMRrn7S20tBgCA2KZPTy7oc5dK6X60Nj9a18wGmdnA+vPc/SN3v8fdvy1pBIEPAChWVfX2n2tq4tWRD9k8WreHmf3Qgu6SzpF0WnP960buAwCgWPXvH6bz5sWtI9eyGYZXki5SOF9/kcLV+evMbJakwQrn+utbL+khd78qV0UCAJCm9evD9OCDC/e8fnu0uqfv7l8qBPmazKs2876/pFMUbuE7o970TUkXm1lFnmoGACCvfvjDpD1rVrw6cq21sfePMLMDFUbhq1LYs/+Xus8zI/B9mZmuyUz/W9LRCo/kBQCg6Fx1VdJ+8sloZeRca3v690u6T9IgSZMlHSLp5Ja+4O5/dfcZ3tq9gAAAFLBx48L08svj1pFLrY29P9DdN5f0scLY+/dJurq57rktDQCAeP7P/0nay5fHqyOXsrl6v0Lhgr9OkroqDL3bKXxkV0jqX39a98pn0QAA5Ns++yTt3/42Xh25lM19+t0zry8kvZppd1UYc3+wwimA/gpHAQZm5m2Rj2IBAEjTXnuF6TnnxK0jV7IZke8LMztP0jp3v8vMxkn60N1nmdlZkrZx98vyXikAACm79FJp7Fhphx1iV5Ib2Y7I9x1J75vZSZJ+LektM7tH0qWS/pKn2gAAiGrXXcP0nXfi1pErLe7pm9mJCvfm36tw//2OmY8Ol7Rc4aK+XmZ2dL2vVSjc4jfd3TfkvGIAAFLSu3fSnjlTGjkyXi250Nrh/askrVW4Mt8VLuIzSQ9nPv9A0qrMvPq/2U3SY5nPAAAoSgPrPWlm332Lf3S+1m7Z29Hdd5U0StJMSZcohP/xkv5XUj9JD0mqcvfdMq9d3H27zNP5AAAoaj/6UdJ+9NF4deSCZTOGjpk9rnD1/gRJH0mqdPfVZlYl6X8UhuYdGWNAnqqqKq+urk57sQCAMlL/8bqFsLdvZrPcvar1ng1l+8Cd093908yCtnb31ZLk7tVmto+kvRiBDwBQqp5+WjrkkNB2b7gRUEyyunq/LvAz7QWNPqt197/mujAAAArFwQcn7fqD9hSbbG/ZAwCgrI0ZE6avvhq3jo4g9AEAyMINNyTtYj2hTegDAJCFbbZJ2jNmxKujIwh9AACy0KVL0q67qK/YEPoAAGTp+99P2hs3xqujvQh9AACydMstSfu55+LV0V6EPgAAWaqoSNrFeF6f0AcAoA3OPTdMr7kmbh3tQegDANAGe+6ZtIvt1j1CHwCANjjllKRdUxOvjvYg9AEAaINu3aThw0N7xIi4tbRV6qFvZneZ2Uwzm9hKv8Fm9npadQEAkK0jjgjTpUvj1tFWqYa+mY2VVOHuIyUNN7NtW+h+o6Qe6VQGAED2Jk1K2v/4R7w62irtPf3RkqZl2k9J2r+pTmZ2kKQvJC1KpywAALLXt2/SfvTReHW0Vdqh30tS3WUPSyUNbtzBzLpK+omkS5v7ETMbb2bVZla9ePHivBQKAEBL6sbirz9gT6FLO/RXKTlkX9nM8i+VdKu7L2/uR9x9qrtXuXvVoEGD8lAmAAAtO+ecMH3zzeK5dS/t0J+l5JD+CEnzm+hziKTzzexZSbua2Z3plAYAQPYuuCBpV1fHq6Mt0g79RySdamZTJJ0gaa6ZNRjTyN1Huftodx8t6W/ufnbKNQIA0KpevaTevUP7mWfi1pKtVEPf3VcqXMz3sqQD3f0Nd2/21r1M8AMAUJB23TVMX345bh3ZSv0+fXdf5u7T3J0r8wEARe2b3wzTd9+NW0e2GJEPAIB2OvTQMH3rLWn9+ri1ZIPQBwCgncaMSdrXXRevjmwR+gAAtFNFhbT99qF9xRVxa8kGoQ8AQAf87Gdh2qMIBo4n9AEA6IC99w7TL78s/EF6CH0AADqgX7+kvWJFvDqyQegDANABZtLQoaE9Z07cWlpD6AMA0EEbNoTp+efHraM1hD4AAB101llh+uabUk1Ny31jIvQBAOigK69M2sOGxaujNYQ+AAAd1K2b9P3vx66idYQ+AAA5MLHZx8cVDkIfAIAc2GyzpH3OOfHqaAmhDwBADphJI0eG9owZcWtpDqEPAECOTJgQppWVcetoDqEPAECO1D18Z86cwhySl9AHACBHtt02aT/wQLw6mkPoAwCQIz17Ju1nnolXR3MIfQAAcuj008P07rujltEkQh8AgBwaMyZp19bGq6MphD4AADn0b/+WtP/nf+LV0RRCHwCAHNt88zB99dW4dTRG6AMAkGN1e/v33x+3jsYIfQAAcuy005L2a6/Fq6MxQh8AgBzbbbek/ctfxqujMUIfAIA8OO+8ML3ttrh11EfoAwCQB+PGJe0bb4xXR32EPgAAeTB6dDJC30svRS3lK4Q+AAB5cu+9YfrII3HrqEPoAwCQJ7vvnrT//vd4ddQh9AEAyJPhw5tux0LoAwCQRxMnJu05c+LVIRH6AADk1dVXJ+3Jk+PVIRH6AADkXd2wvPPnRy2D0AcAIN+OPjpMY9+6R+gDAJBnY8cm7TVr4tVB6AMAkGeVlUn700/j1UHoAwCQgu22C9MVK+LVQOgDAJCCfv3C9FvfilcDoQ8AQAoGDw7TlSuldevi1EDoAwCQgoceStqLF8epgdAHACAFvXpJW20V2mvXxqmB0AcAICVdu4bpsmVxlk/oAwCQknnzwjTWbXuEPgAAKTnmmDB95ZU4yyf0AQBIyRdfhOlrr8VZPqEPAEBK9tsvTD/6KM7yCX0AAFKy/fZhOndunOUT+gAApGTXXeMun9AHACAlW26ZtN3TXz6hDwBASnr0SNoLF6a/fEIfAICUmCWP2X366fSXT+gDAJCi/v3DdPbs9JdN6AMAkKKjjgrTW25Jf9mEPgAAKTrwwKRdW5vusgl9AABSdPzxSfvmm9NdNqEPAEDKBg8O05/9LN3lph76ZnaXmc00s4nNfN7XzB43s6fM7Pdm1jXtGgEAyKezzgrTiop0l5tq6JvZWEkV7j5S0nAz27aJbqdImuLuh0laJOnbadYIAEC+nXRSmC5YkO5yO6e7OI2WNC3TfkrS/pLeq9/B3W+t93aQpH+mUhkAACkZOjRpu4f799OQ9uH9XpJqMu2lkgY319HMRkrq7+4vN/HZeDOrNrPqxYsX56dSAADyZODApP3kk+ktN+3QXyWpbhDCyuaWb2YDJP1c0plNfbaTKs0AAAi/SURBVO7uU929yt2rBg0alJdCAQBIw+TJ6S0r7dCfpXBIX5JGSJrfuEPmwr3pkn7s7imf7QAAIB2XXx6mS5akt8y0Q/8RSaea2RRJJ0iaa2bXNOpzlqTdJV1uZs+a2Ykp1wgAQN4dcECYzpmT3jJTvZDP3Vea2WhJh0qa7O6LJL3RqM9tkm5Lsy4AANK2/fZJe9myZEz+fEr9Pn13X+bu0zKBDwBAWdpii6S9KKVEZEQ+AAAiqaoK03vuSWd5hD4AAJF8+WWYpnUFP6EPAEAk112XtCdNyv/yCH0AACIZMyZpT2zyiTS5RegDABDR7NlJ+/bb87ssQh8AgIh22y1pn3tufpdF6AMAENkf/pC0ly7N33IIfQAAIjv66KS9ySb5Ww6hDwBAAah/294//pGfZRD6AAAUgIsvTtoHHZSfZRD6AAAUADNp/PjQfvddqbY298sg9AEAKBD//d9J+/DDc//7hD4AAAWislIaNSq0Z8zI/e8T+gAAFJCpU5P2hg25/W1CHwCAArLddkn7+edz+9uEPgAABcRM2myz0D711Nz+NqEPAECBOeaYMK2pye3vEvoAABSY+o/ZXbgwd79L6AMAUGD69UuG491ii9z9LqEPAEABOuuspP3ii7n5TUIfAIACdP314aI+qeHh/o4g9AEAKFAnnhimTzyRm98j9AEAKFD/9V9Je+bMjv8eoQ8AQIHabjupb9/Q3nffjv8eoQ8AQAF7+OGk/cEHHfstQh8AgAJ28MHJBX1jx3bstwh9AAAK3MSJYTpnjvT44+3/HUIfAIACd9llSfvII9v/O4Q+AAAFrnt3ae7cjv8OoQ8AQBHYaSfpoos69huEPgAARWLKFOnTT9v/fUIfAIAi8o1vtP+7hD4AAGWC0AcAoEwQ+gAAlAlCHwCAMkHoAwBQJgh9AADKBKEPAECZIPQBACgThD4AAGWC0AcAoEwQ+gAAlAlCHwCAMkHoAwBQJgh9AADKBKEPAECZIPQBACgThD4AAGWC0AcAoEwQ+gAAlAlCHwCAMkHoAwBQJgh9AADKBKEPAECZIPQBACgTqYe+md1lZjPNbGJH+gAAgLZJNfTNbKykCncfKWm4mW3bnj4AAKDt0t7THy1pWqb9lKT929kHAAC0UeeUl9dLUk2mvVTS7u3pY2bjJY3PvF1rZm/muE583UBJn8UuosSxjvOPdZx/rON0bN+eL6Ud+qsk9ci0K9X0kYZW+7j7VElTJcnMqt29Kveloj7Wc/6xjvOPdZx/rON0mFl1e76X9uH9WUoO14+QNL+dfQAAQBulvaf/iKQXzGxTSUdIOsnMrnH3iS302SflGgEAKEmp7um7+0qFC/VelnSgu7/RKPCb6rOilZ+dmodS8XWs5/xjHecf6zj/WMfpaNd6NnfPdSEAAKAAMSIfAABlomhCn5H88q+19Wdmfc3scTN7ysx+b2Zd066xFGT7d2pmg83s9bTqKiVtWMe3mtlRadVVSrL4/0V/M/tfM6s2s9vTrq9UZP4/8EIrfbLOvqIIfUbyy78s198pkqa4+2GSFkn6dpo1loI2/p3eqOT2VWQp23VsZt+SNMTd/5hqgSUgy3V8qqQHMrfv9TYzbuNrIzPrL+lehfFrmuvTpuwritAXI/mlYbRaWX/ufqu7/znzdpCkf6ZTWkkZrSz+Ts3sIElfKGxcoW1Gq5V1bGZdJN0hab6ZHZNeaSVjtFr/O14iaWcz6ydpc0kL0ymtpGyQdKKklS30Ga02ZF+xhH7jUfoGt7MPmpf1+jOzkZL6u/vLaRRWYlpdz5nTJj+RdGmKdZWSbP6WT5P0lqTJkvYysx+kVFupyGYdvyhpS0kXSno70w9t4O4rs7iDrU3ZVyyhn5OR/NCirNafmQ2Q9HNJZ6ZUV6nJZj1fKulWd1+eWlWlJZt1vJukqe6+SNKvJB2YUm2lIpt1fKWkc939p5LekXRGSrWVmzZlX7EEIyP55V+r6y+zBzpd0o/dfUF6pZWUbP5OD5F0vpk9K2lXM7szndJKRjbr+H1JwzPtKkn8PbdNNuu4v6RdzKxC0t6SuD88P9qUfUVxn76Z9ZH0gqS/KDOSn6Rx9Qf2aaLPPlkcFkFGluv43yX9X0lvZGbd5u6/SbvWYpbNem7U/1l3H51ehcUvy7/l3pLuVjgU2kXS8e5e08TPoQlZruO9JP1S4RD/TEnHufuqCOUWvbr/D5jZTpJO7kj2FUXoS19dxXiopOczh+Ta1QfNY/2lg/Wcf6zj/GMdF462/LcomtAHAAAdUyzn9AEAQAcR+gAAlAlCH4DMrFfmKmsAJYzQByCFe31rzcyzeN1d9yUzG5Xld+q/No/47wTKWufYBQAoCFtIWitpXeb9+5KmSLq1Ub9nJX1S7/36zLR/lst4o953AKSM0Acgd/9qXHQz21PSJpL+2HhUQDMbKumjerNqM99vdfTAzBjsX30HQPo4vA+gsSskveju/6/+TDPrrvCgpQ/qzV7fqM9nTRzOf63R7xP6QCSEPgBJX13Md5+kgySdV2/+gMxT//5LYSjVOS38zGpJB7q7ubtJukjSl3ksG0AbcHgfKHNmNkzSCQoBvVHS4Y328jdIelLhYr9J7t7SI5U3ZjkPQASEPlDGzKybwjO4Oyk8ZvZOd2+wZ+7uK8xsiLsvyeYns5wHIAIO7wNlzN3XKjygYwdJx0ha3dRtdpLqn6s/uYWf7C7pmXrfuykzD0ABYE8fKHPuvjLTXC3pYUkXt9B9jqQ1LXy+nb6+Z8+Fe0CBIPQB1NkoaZW7z2+ug5ltVAvn6HmcNVDYOLwPoCPas+PAcL9AJOzpA6jve2b2vVb61P//RhdJypy/z1aXNlcFICfY0wdQxyX9SmFI3eZeK9XwwrzOklbU3Zff0kvS1vW+AyACc2/LBjoAJMyss6Re2ZzLN7NOkvoobCTwPx4gAkIfAIAyweF9AADKBKEPAECZIPQBACgThD4AAGWC0AcAoEwQ+gAAlIn/DzU2eyWE405XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_precision_vs_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, \"b-\", linewidth=2)\n",
    "    plt.xlabel(\"召回\", fontsize=16)\n",
    "    plt.ylabel(\"精度\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_precision_vs_recall(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 通过选择阀值来实现最佳的精度/召回率权衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目标设定为90%的精度，阀值大概在30000左右 , 设置了阀值为30000\n",
    "y_train_pred_90 = (y_scores > 30000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9143861410996736"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6718317653569452"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "* 获得了一个90%精度的分类器，但如果召回太低，精度再高，也不怎么有用\n",
    "* 如果工作中，需要99%的精度，你应该回应，召回率是多少？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC 曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 本质是 真正类率tpr和假正类率fpr（错误的分为正类的负类实例比例）\n",
    "# 与召回/精度曲线非常相似"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thresholds = roc_curve(y_train_5, y_scores)  # 真正  假正 阈值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3xUVeL+8c+ZSQ8JCRA6IiiKUgIhBgUEBEEsW1yVIkVQFHV1d3W/6qqgLKJrW/Rrw1VBQBcQ92fZrwURUKyUgMYCKkoH6ZDeZub8/pgBAYEESOYmd5736+WLyeQy8yTL5sk599xzjbUWERERcT+P0wFEREQkPFT6IiIiEUKlLyIiEiFU+iIiIhFCpS8iIhIhVPoiIiIRQqUvIiISIcJe+saYRsaYjys4Zoox5nNjzNhw5RIREXG7sJa+MSYVmA4kHuWYPwBea+05QGtjTJtw5RMREXGzcI/0/cAgIO8ox/QG5oQezwN6VHMmERGRiBAVzjez1uYBGGOOdlgisDn0eDeQcegBxpjrgOsAEhMTu7Rt27Zqg4qIyAmxFiwWC2DBF7D4A5ZAIPQclnL/L9vAW8AfsPj8Abwegz3gE/seF5X5iIny/Or5knI/HmMo9QXwGPCYA/7+/mN//V5Oi/IY9sWIjfJgDJT5AiTGRmGAMn+AhJhgTfvLy9i9bRPlJcUkJKdSlLdnp7U27Zjfsyq/gCpSAMSHHtfhMLMR1trngOcAMjMzbXZ2dvjSiYjUAIGAJa+knJLyALsKS7GhYi3zBdieX0KM10PAgrUWv7UELGzcXURijBcL+z8XCH0uYC2rtxXQoE7M/o8DgeDfzdmYS4t68fhDxf3lxlwaJcfiMSZY5Nby3dZ8kmKjyC/1HffX5AFijvL5ukd4PuG43/EXzVLiMaFfGLwegzGwZkchWa3qsXlPMZ1PSiEu2kuUx7Atr4QW9RJo37QupT4/zVMTiPZ68HgACw2SYomP9pIUF0VslPeg1/WYCge+v1JYWEjr1q2Jj4nmpWkvMmjQIIwx64/n66yJpb+c4JT+YiAd+N7ZOCIih1fuD1DuD4SKL1jEAWsp8QUoKvXhtxaf31JQ6qOozIcxBmst1rK/WPd9vHZXIXVio/hqUy57i8qJjfbgMYZAwFLqC7Bs3W5aNUjky417aZQcy7a80rB+rV9vzj3o450Fv37/Qws/PtqL1xMsu9zicholx7KnqJzMlqnEhT63ZkcBZ51cD6/HEOUx5Jf6SIqNIi0pFmMMHmP2j94BCkp9tKyfEHze88vnynwBmqfGE+X1UC8hBmMI/RcqWn4pXGPAAKkJMXg8x1bA4VReXk50dDSJiYk8+eSTnHXWWbRq1eqEXtPR0jfGnAlcaa09cJX+G8DHxpimwIXA2Y6EE5FaoajMR0GJj73F5RSX+fEFAmzNLcVjoDxg2bi7iNgoD9aC3wZHqit/zqNhUiyBgMUXsHy9OZdGyXEYOGjkG7CwbO1uWqcl4g9Y8orL2ZJbQnJcFHklxz+iPV5fbtwLcFDhN6gTy86CUpqnxlMvMWZ/wcZ4PZxUb185EipPw7pdhWSclLp/9Lm/UEN/bs0t4cwmyfsLNTg6NeSX+GjVIAGvx4PXA/4ANEyK3f95jwdio7ykJkQTF+0lLtob9u+Pm3z99dcMHjyY8ePHc8UVVzBw4MAqeV1HSt9a2zv050pg7CGfyzPG9Ab6AQ9ba3N/9QIiUqv5A5ZdBaVszy+luNy//1yv31ryS3zkFZezq7CMHfmlbMsr4cftBTRNCU4v+wIBFq/ZXQ2pjvyj5tstB689PrTwE2O8oZL8ZQp3Z0EZpzdKCo5gvYafthfQsXnK/hH8/lEn+85BW3YWlNGxeV2KSv00So7l5AaJxER5iPIYwBAb7aFBYizJ8VEkxUWTEh9do0eqcuystTz77LPceuutpKSkUK9evSp9/Zo4vY+1dg+/rOAXkRrEWsuG3UXsLizDFwhOX2/eW0xMlAefP8AP2wpISYjGHxpl/7AtH6/HsHz9HurXiWVH/vFNS6/eXnDUzzepG8fPuSV0PimFKI9h3a4islrVI9pjWL+7iM4tUon2BqeEvcawI7+UM5ok4fUGSzWvuJyW9RMPGt2aA6aVG9SJJcobfL5+YgwxUR5iojxEe7XHmVSN3bt3c+211/Laa68xYMAApk+fTsOGDav0PWpk6YtI9bPWsi2vlPyScrbmlbCroIzdhWX4A5bV2/P5aUchqQkxLF6zi5MbJODzBxdrnYhDC9+Y4CrvrJPr4fH8UrY78ktpWT8Ba6F5agKN68aSHBdN89SE/eeIE2K8nFQ/gaTYqGNeGCVSE82fP5///ve/PProo9xyyy14PFX/C6VKX6QG+2XRV3Dq21rYU1TG5j3FFJb52ZZXsv989YHnoq217MgPTp/vLiwjPtpL9vo9NKkbx+I1uzipXgLrdhVVOsc3mw+/tUaXlql4PQZscGFXx+Z18XgMuwrKaNc0mShP8DKqlvUTOaNJEnHRXurXiSElPuaXS69EIpjf7ycnJ4eMjAwGDhxIZmYmrVu3rrb3U+mLnAB/wIamuQPsLSqnoNTH5j3Bqe59lzIFL32CtTsLqRsfjS8QLOS9RWUkxHrZkV/KvJXbaJ4aj99v2ZpXQsD+MgquSmt3FgIcVPhRHkNKQgwpCdGk1Ynl1IZ1SIyNIre4nJPrJ3BqwzrEx3hJiY8hymuIi/LSNCWOKE1ri5yQzZs3M2zYMBYvXsz333/PSSedVK2FDyp9iUA2NCL2BYKXWhWU+igpC+ALBMgr8VFS7sfnt+wsKKXMH8Bay8LvtrOzoIyYUNF9vmZXlefauLv4kJzBPw9dZb1vA5K0pFjqJ8aSGOulaUr8Qeeg9x27q7CMxslxNEmJIy10TrpRchzJcdEkx0XTLDU+OFIXkbB66623GDlyJMXFxUyePJkWLVqE5X1V+lKj5ZeU8+H3O7D8spGIPxCcwt60u4jY0GVBPr/l2y25pCXFsj00it5/7bS1lJT7+WHb0ReCnYjGyXF4PYbNe4tp2zgJjzG0apD4y2VPxmCMYfPeItJbBBeaFZb6aVAnhgZ1YmmSEk+DOjGkJMQQ5THEeD0kx0cf10YeIlJzWWu59dZbefzxx+nUqROzZ8/m9NNPD9v7q/QlLKwNbjCyI7+U9buK2Ftcxor1e6kTF8UXG/bQKDlu/yVb63YWkrOp+q/UjPYGF4SV+gJYC60aJBLlMazeXkC3U+oT5fWwIXRNc2x0cBq++6n1adMwidhoD4kxUbSsn0BirP5vJCKVY4zB6/Xypz/9iYceeoi4uLiwvr9+Wkm1KPcHWPVzHrOWbuSLDXtOeNV367RE2jetu3/a2hiD1wMbdhfR+aRUokIruvcWlXNm02TSkmJJio365drp0NR3vcQYUhNiiI/RxiEiEh7WWmbMmEGbNm3o1q0bjzzyiGMzeCp9OSH+gOW7rXk8/cGPbNhdxN6icjbtKT7i8TFeD2X+AKc3SiIpLgqPMZzXtiF5JeWcklYHrwe8Hg/WWk5Jq0PbxklaMCYitVZ+fj433HAD//73vxk+fDjdunVz9JSdSl8qVO4PsGZHIT/nFrN6WwEzl25gZ35ppW+s0bdtQwa0b8xv0ptqa04RiRjZ2dkMHjyYtWvXct9993HnnXc6HUmlL0HWBm9zuXjNLhav2cW6XYXM/Wbr/ts+VqR5ajwD2jWm7xmNiIv20KFZXY3QRSRiLV26lB49etC4cWMWLVpEjx49nI4EqPQjTpkvwKc/7aSo1M+c7I2s/Dmv0tuiJsVGER/jpX+7RpzZpC5ZrerRpG6cFrKJiIQEAgE8Hg9dunRh7Nix3HTTTVW+f/6J0E/rCLCnsIzFa3bxz/d/4McK9i/f5+IOTWieGk96ixR6nZamYhcRqcD8+fO59dZbmTt3Lk2bNuWee+5xOtKv6Ce5i2zeW8z0z9bx6Y87QwvsjrxiPi0plh6nNsBjDH/u24a0pFitaBcROQ7l5eXcc889PPTQQ7Rt25a8vDyaNm3qdKzDUunXUmt3FvLkgtV8uyWP77fl4zFUeP795PoJ3Nj7VK7IbK4NX0REqsDatWsZMmQIS5Ys4dprr+Xxxx8nISHB6VhHpNKvJbbmlvDwe9/x2orNh/38gYXf7ZT6XJbRnIyWqdSvE0OdmCjdc1tEpBpMmDCB7777jldeeYWBAwc6HadCxlb1HT3CLDMz02ZnZzsdo8otX7+bd77eypRP1h71uE4tUrix9ymc1iiJRslxmqIXEalmRUVF7N69m+bNm7N371727NlDq1atwprBGLPcWpt5rH9PI/0aZNEPO3htxSbe/HLLYT8fE+WhZ5s0bjzvFDo1T9HoXUQkzL7++msGDRpEYmIiS5YsISUlhZSUFKdjVZpK32Hl/gAT31rJa19sJr/k15vdXNKxCT3bpPHbTtrYRkTEKdZann32WW655RZSU1N54okn8Hhq314kKn0HWGv5YVsBb3y5mckf/nTQ5xJjvAw9uyVXZp3EyQ0SHUooIiL75ObmMmrUKF5//XUGDBjA9OnTadiwodOxjotKP4zW7CjgpplfsPLnvF99rnVaIjOuzqJ5as1d9SkiEoliYmJYv349jz76KLfcckutHOHvo9KvZrlF5SxavYO//b+vKCrzH/S5Ds3qUu4PMOOaLBomhff2iiIicmR+v5+nnnqKq6++mqSkJJYsWUJUVO2vzNr/FdRA1lqmfbaOR977/ldFD3Bp52ZM+F07kuKiHUgnIiJHs2nTJoYNG8aiRYuIi4tjzJgxrih8UOlXuS837mXQvz6n1Bc46PnTGyUxqvvJDM46yaFkIiJSkf/7v/9j1KhRlJSUMG3aNEaMGOF0pCql0q8C1loW/bCD619eTkn5L2WfHBfFzX3acHWPVnh1eZ2ISI329NNPc9NNN9G5c2dmzZrF6aef7nSkKqfSP0Hb80ro/eiHv5rGf/OP3UlvUXuu3RQRiXSXXHIJGzZsYMKECcTGxjodp1rU3iWINcDXm3Lp99hH+wvfY+BPfU7lpwcuUuGLiNRw1lqmTZvGoEGDCAQCtGzZkoceesi1hQ8a6R8Xnz/A3/9vJS8tXg9AQoyXF67KpNspDRxOJiIilZGXl8cNN9zAzJkz6d27NwUFBSQnJzsdq9qp9I/Rpj1F9Hjog4Oee/tP59JKG+mIiNQKy5YtY8iQIaxbt4777ruPO++8E683MnY8Vekfg6827eW3T326/+P0Fim8fkM37YEvIlJLlJeXM3DgQAKBAIsWLaJ79+5ORworlX4lfLc1jwGPf3zQc9OvzqLXaWkOJRIRkWOxY8cOUlNTiY6O5vXXX6dly5akpqY6HSvstJCvAtvzSn5V+NNGnaXCFxGpJd5//306dOjAhAkTAOjUqVNEFj5opH9UL3y8holvr9r/sS7DExGpPcrLyxk3bhwPP/wwbdu25YorrnA6kuNU+kfw9lc/H1T4b93cg/bN6jqYSEREKmvt2rUMGTKEJUuWcO211/L444+TkKAbmqn0D1HuD/Dkwh95YsHq/c/9eP+FRHl1JkREpLbYs2cPa9euZc6cORrhH0Clf4DXVmzi1jk5Bz23fOz5KnwRkVqgsLCQN954g6FDh5KRkcHatWs1uj+E2izkx+0FBxX+4LNasHLCBdSv496dmURE3OKrr74iMzOT4cOH8+233wKo8A9DI32g1Ofnimc/2//xqgkDiI+JjI0aRERqM2stkydP5tZbbyU1NZX333+fdu3aOR2rxlLpA89/tIY9ReUALPxrLxW+iEgtMXLkSGbMmMGFF17ItGnTaNiwodORarSIL/1vt+Ty6LwfAHh8UCdap9VxOJGIiFRW//79SU9P5y9/+Qsej85YVySiS3/9rkIufuITAM5t04Dfd27mcCIRETkav9/P/fffT5MmTbj22msZOnSo05FqlYj9tchay7ApS/Z//OgV6Q6mERGRimzatIm+ffty7733smzZMqfj1EoRO9L/95INbNxdDMBrN3ajUXKcw4lERORI/vvf/zJq1ChKS0uZPn06I0aMcDpSrRSRpb+3qIz7Q7vt3drvNDJOisw9mEVEaoNVq1bx+9//nk6dOjF79mxOO+00pyPVWhFZ+v+c9wPF5X5a1k/g5j6nOh1HREQOIzc3l7p163LGGWfw+uuvM2DAAGJjtXfKiYi4c/plvgD/b8UmAIaf3RJjjMOJRETkQNZapk2bRsuWLfnss+AeKr/73e9U+FUg4kr/iQWrKSrzA3B191YOpxERkQPl5eUxdOhQRo0aRefOnWnZsqXTkVwl4kr/qQ9+BIKjfI9Ho3wRkZpi2bJldO7cmTlz5jBx4kTmz59Ps2a6lLoqRdQ5/XU7C/c/HtOrtYNJRETkUO+//z4+n49FixbRvXt3p+O4UkSN9O987WsAurRMpXmqbsQgIuK0bdu28fnnnwNwxx13kJOTo8KvRhEz0i8u8/P5ml0ADMps4XAaERGZN28eI0aMICYmhh9//JGYmBhSUlKcjuVqETPSn7dy6/7HV2Q2dzCJiEhkKy8v529/+xsXXHAB9evX5+233yYmJsbpWBEhYkb6/7tgNRAc5esyPRERZ+Tl5dG/f3+WLFnCddddx2OPPab73odRRIz09xaVsWZHcBHftT21gE9ExClJSUm0b9+eOXPm8K9//UuFH2YRUfr/XrIBgGYp8ZzaULfOFREJp8LCQv74xz+yevVqjDG88MILXHHFFU7HikgRMb2/t6gMgPp1dM5IRCScvvrqKwYNGsT3339Px44dadOmjdORIlpEjPSXrtsDwOhzNbUvIhIO1lqefvppsrKyyM3NZf78+YwZM8bpWBEv7KVvjJlijPncGDP2CJ9PNca8Y4zJNsb860Tfb2dBKTkb9wJwdut6J/pyIiJSCf/617+46aab6Nu3Lzk5OfTp08fpSEKYS98Y8wfAa609B2htjDncPM9w4N/W2kwgyRiTeSLv+eaXWwDofFIKDZPiTuSlRESkAqWlpQBcddVVTJkyhbfeeou0tDSHU8k+4R7p9wbmhB7PA3oc5phdQHtjTArQAth4Im/43rfB6/Nb1tMKURGR6uL3+/n73/9Op06dyM/PJz4+nquvvlqXSNcw4S79RGBz6PFuoNFhjvkEaAn8CVgVOu4gxpjrQtP/2Tt27Djim1lr+Tm3GIAB7ZucWHIRETmsTZs20adPH8aPH09m5glNzko1C3fpFwDxocd1jvD+9wLXW2snAN8Bow49wFr7nLU201qbebRpo8/X7GLj7mLqxkfT94yGJ55eREQO8uabb5Kens7y5cuZPn06L730EklJSU7HkiMId+kv55cp/XRg3WGOSQU6GGO8QFfAHu+bLVy1HYCep6UR7Y2ICxVERMImEAjwyCOP0LJlS1asWMGIESOcjiQVCPd1+m8AHxtjmgIXAoONMROttQeu5P8H8CLBKf7PgVnH+2bvfhM8n39h+8bHHVhERA723XffUb9+fdLS0njttdeoW7cusbGxTseSSgjr8Ndam0dwMd9i4Dxrbc4hhY+1dqm1tp21to61tp+1tuB432/z3uD5/Jb1tYhPROREWWuZOnUqXbp04dZbbwWgYcOGKvxaJOxz3tbaPdbaOdbarRUfffx8/sD+x60aJFbnW4mIuF5ubi5XXnkl11xzDV27duWhhx5yOpIcB9ee6P4pdIMdgISYiNhtWESkWnz77bdkZGTw6quvMnHiRN5//32aNm3qdCw5Dq5tw1ezg5f39zvzcFcFiohIZTVs2JBGjRrx0ksv0a1bN6fjyAlw7Uh/8dpdAHRoVtfhJCIitc+2bdu4/fbb8fl8pKWl8emnn6rwXcC1pb+3qByATi1SHE4iIlK7zJs3j44dO/Lkk0+yYsUKAO2s5xKuLH1rLZv2BFfut9dIX0SkUsrKyrj99tu54IILSEtLY9myZWRlZTkdS6qQK8/p7yt8gHqJMQ4mERGpPUaNGsXMmTMZM2YMkyZNIiFBlzu7jStLf+na4Hb97ZslO5xERKTm8/v9eL1e/vrXv/KHP/yByy67zOlIUk1cWfrrdxcB0LK+rs8XETmSwsJC/vSnPxETE8PkyZPJyMggIyPD6VhSjVx5Tn/hd9sAyDgp1eEkIiI1U05ODpmZmbz44ovUq1cPa4/7NidSi7iu9AMByzeb8wD4TUfdTldE5EDWWp566im6du1Kbm4u8+fP5/7779fq/AjhutJf+XPe/scNk+McTCIiUvNs3ryZO++8k759+5KTk0OfPn2cjiRh5Lpz+ovXBDflOameVp2KiOzzzTff0K5dO5o3b87SpUtp27atRvcRyHUj/fW7gov4GiXrrk8iIj6fj/Hjx5Oens7LL78MwBlnnKHCj1CuG+l/sXEPAF1b1Xc4iYiIszZu3MjQoUP5+OOPGT58OL///e+djiQOc13pez3ByYsW9eIdTiIi4py3336bESNGUFpayowZMxg+fLjTkaQGcF3p5xcH99zX9rsiEulatWrFrFmzaNOmjdNRpIZw1Tn9QMCyZmchAE3qaqQvIpFl1apVvPjiiwBcfPHFLFmyRIUvB3FV6X+3NX//49SEaAeTiIiEj7WWKVOmkJmZyd13301BQQEAXq/X4WRS07iq9JdvCC7ia5Qcq5WpIhIRcnNzGTJkCKNHj+bss88mOzubOnXqOB1LaihXndPftCd4ud6pDfUPXkTcr7S0lKysLH766SceeOABbr/9do3u5ahcVfo5G/cCkN48xeEkIiLVx1qLMYbY2FhuueUWOnbsSLdu3ZyOJbWAq6b3t+wtAaBZqhbxiYg7bd26lQsvvJB3330XgOuvv16FL5XmqtJPiAlOa52apul9EXGfefPmkZ6ezqJFi9i5c6fTcaQWck3pW2v3r94/Ref0RcRFysrKuP3227ngggtIS0sjOztbm+3IcXFN6eeGNuUBqJ8Y42ASEZGq9eabb/LII49w/fXXs2zZMtq1a+d0JKmlXLOQb2dBKQDNU+N1uZ6IuMKGDRs46aSTuPzyy/n000917l5OmGtG+rsKygBomKS764lI7VZYWMjVV19Nu3btWLt2LcYYFb5UCdeM9HcVBku/XqJKX0Rqry+//JLBgwfzww8/cNddd9GiRQunI4mLuGakv2ZHcNvJFG2/KyK11FNPPUXXrl3Jy8tj/vz5TJw4kago14zNpAZwTen/nBu8Rj+/pLyCI0VEaqavv/6afv36kZOTQ58+fZyOIy7kml8ht4ZKv3lqgsNJREQqb9GiRSQnJ9O5c2eefPJJoqOjtRhZqo1rRvrR3uCX0jRFu/GJSM3n8/m499576dOnD2PHjgUgJiZGhS/VyjUj/e+25gHQsp5G+iJSs23cuJGhQ4fy8ccfM2LECJ566imnI0mEcE3pJ8YGv5Qor35LFpGa65tvvqFnz56Ul5fz0ksvMWzYMKcjSQRxzfT++l3B2+o2So5zOImIyJG1bduWQYMGsWLFChW+hJ1rSr+g1AdAnVjXTF6IiEusWrWKCy+8kJ07dxIVFcXkyZNp06aN07EkArmi9AMBu/9xw2RtziMiNYO1lilTppCZmcny5cv56aefnI4kEc4VpV/qC+x/HBvldTCJiEhQbm4uQ4YMYfTo0Zxzzjnk5OTQtWtXp2NJhHNF6ReWBaf2U7Ubn4jUELfddhv/+c9/eOCBB5g3bx5NmjRxOpKIO1bv54Vuq7unSLvxiYhzAoEAubm5pKamcv/99zNq1CjOOeccp2OJ7OeK0i/3B8/pt6injXlExBlbt25lxIgRFBUV8eGHH5KWlkZaWprTsUQO4orp/VKfH4C68ZreF5Hwe++990hPT9+/2Y7Xq7VFUjO5ovR3FpQCEON1xZcjIrVEWVkZt912GwMGDKBhw4ZkZ2dz3XXXaStdqbFc0ZJbc4Olv7OgzOEkIhJJSktLeeONN7j++utZunQp7dq1czqSyFG54pz+Po21G5+IhMGbb75J//79SUpKYvny5SQnJzsdSaRSXDHSX7OjAIB2zfR/PBGpPgUFBYwaNYrf//73PP300wAqfKlVXDHSrxMX/DI0vS8i1eXLL79k0KBBrF69mnHjxvGXv/zF6Ugix8wVpV9cHly937ZxksNJRMSNXnnlFUaMGEGDBg1YsGAB5513ntORRI6LK6b3V27JAyA2yhVfjojUMBkZGVx66aXk5OSo8KVWc0VL7lvAVxIa8YuInKhFixbxpz/9CWstbdq0Yfbs2TRo0MDpWCInxBWlXxK64U7z1ASHk4hIbefz+bj33nvp06cPc+fOZdeuXU5HEqkyrij9nI17AYiLdsWXIyIO2bhxI+eddx4TJkxg+PDhrFixQqN7cRVXLORrnhrPht1FWOt0EhGprfx+P+effz5btmzh5ZdfZujQoU5HEqlyrij9/JLgrXWbpOiGOyJybEpKSoiOjsbr9fLcc8/RrFkzTj31VKdjiVQLV8yHF5QGSz8pzhW/w4hImKxatYqsrCweeeQRAHr16qXCF1dzRenvW7UfF607W4lIxay1vPDCC3Tp0oWtW7eSnp7udCSRsAh76RtjphhjPjfGjK3guGeMMb+pzGv+nFsC6Dp9EalYbm4uQ4YM4dprr6Vbt27k5ORw4YUXOh1LJCzC2pLGmD8AXmvtOUBrY0ybIxx3LtDYWvt/lXndfWWfEKORvogc3cqVK3njjTd44IEHmDdvHk2aNHE6kkjYhHto3BuYE3o8D+hx6AHGmGjgeWCdMeZ3lXnR0tB1+nFRKn0R+bVAIMDChQsBOOecc1i3bh133nknHo9mByWyhPtffCKwOfR4N9DoMMeMAFYCDwNZxpibDz3AGHOdMSbbGJO9Y8cOAKK9Bo/HVE9qEam1fv75Z/r370/fvn1Zvnw5AI0bN3Y4lYgzwl36BcC+6+rqHOH9OwPPWWu3Ai8Dv9ro2lr7nLU201qbWa9+cOOMcr8u0heRg82dO5f09HQ+++wznn/+eTIyMpyOJOKocJf+cn6Z0k8H1h3mmB+B1qHHmcD6o73gvg15NMgXkQONHTuWCy+8kMaNG5Odnc3o0aMxRj8oJLKdcOkbYzyhhXeV8QYw3BgzCRgIfGuMmXjIMVOA84wxHwE3Ao8e7aBGjkYAACAASURBVAUDodZvqo15ROQALVq04MYbb2TJkiWceeaZTscRqREq3M3GGBMD/BV4EIiz1haHno8DBhFcmPceUOHdbqy1ecaY3kA/4OHQFH7OIcfkA1dU9gvwB4KlX+4PVPaviIhLzZw5E6/Xy6BBgxgzZozTcURqnMqM9D3AbcDNwD0HPP8ycBdggPLKvqG1do+1dk6o8E+YJVj6ZT6VvkikKigoYOTIkQwdOpTp06djdSMOkcOqzL61ZUAh8A6QbYz5HGhD8PK7LtbaImOMYzeyDw30Ob1xklMRRMRBX3zxBYMHD2b16tWMGzeOe+65R+fuRY6gwtK31gaMMeXW2h+NMbcAG4AvgKXA74wxc47+CtXLF5rW92n1vkjEWbNmDWeffTZpaWksXLiQ3r17Ox1JpEY71jvUbLXWfmmM6Qw8AZwJfF71sSpv32/05QGVvkik8Pl8REVF0bp1a/73f/+Xyy+/XPe9F6mESq/eN8ZkAf/PGDOA4KV0a4Bt1tplBM/rO2Lf6v3WDRKdiiAiYfThhx9y2mmn8cUXXwBw/fXXq/BFKumopW+MOdsY82bowy+ARwhedreb4Ar71NDld/HGmEmh/x43xjxbrakPUO4Llr5utiPibj6fj3vuuYc+ffoQHR2tLXRFjkNF0/utCW6dGw28DowH/kzwWnoL5AGnEPzloVXo73iBuGrIelj7/n+/aU9xuN5SRMJsw4YNDB06lE8++YSRI0fy5JNPUqdOHadjidQ6Ry19a+1MYKYxZhPBgn+IYNn3Bd4keG3+NcBqa+2l1Zz1qE5rpNX7Im714osvkpOTw8svv8zQoUOdjiNSa1V2fqzMWnslsAeoC5QAlwPJQEvAsVV0+y7HjfbqEh0RNykuLmblypUA3HXXXXz11VcqfJETdKwnxZ4FzgB2EZz6z7TWLq/yVMdg328bXm2+L+IaK1eupGvXrvTv35/i4mKio6M5+eSTnY4lUutVWPomeE1crDGmHjCb4Pn9RIKX7DWs3ngV27fzVpRXi3pEajtrLc899xyZmZls27aNF154gfh43VdDpKpU5jr9WILn7gcAs6y13wAYY0YAM4wx3YCY6otYOVEa6YvUasXFxVx11VW8+uqrnH/++bz00ku6771IFavM8NgH3ERwlP+3fU9aa98FHgcCBH8xcESp9twXcYXY2FhKS0t58MEHee+991T4ItWgMtvw+oB/hz4sPORz/whN/3ephmyVEh2a1t9bVOl7/ohIDREIBJg0aRIDBw7kpJNO4o033tC++SLV6IRPhNugr6oizHG+PwDNU3XeT6Q2+fnnn+nfvz+33XYb06dPB1Dhi1SzSpW+MSbWGPOaMSY29HEDY0xDY0yiMcZvjEk84NgZxpju1RX4SKJ0yZ5IrfHuu++Snp7OZ599xvPPP8/YsWOdjiQSESrahndfkwaA34X+BJgKvAeUE9x3vzR0fDIwGGhaHWEPZ98lex6NEERqhdmzZ3PRRRfRuHFjsrOzGT16tEb4ImFS0Uj/TWPMb6215QDW2nJjzLUEV/L/1VpbFnza+kLHjyC4gc8b1Zb4EPs259HqfZGabd+puIsvvph77rmHJUuWcOaZZzqcSiSyHLH0jTEegjfZmRW6PA9jTAvgn8Dt1tqFhxwfB/wFuHffLwnhpM15RGqul19+mR49elBcXExSUhJ///vfdf29iAOOWPrW2oC19l6Cd9MbHnr6CWCJtfbxw/yVfwA/A89VecqjKC7zAyp9kZqooKCAkSNHMnz4cDweD/n5+U5HEololblk7x3gHWNMALgDKIDg+X4bnK8zxph/Ar8HzrbWhvXC+djo4O8tBaW+Co4UkXD64osvGDx4MD/++CP33HMP48aNIyqqMvuBiUh1Oer/A40xc4Gi0IcWeBDwhFbx7zXGZIU+9xvgHGvttmpLegT7zuk3TArb3XxFpALWWm688UYKCwtZuHAhvXr1cjqSiFDxSH8FoZX5BEfyZwCvENx2dwvwGfC/QHPgHmPMn8N9Pt+G1u9rIZ+I83bu3ElUVBQpKSnMnDmTpKQkGjRo4HQsEQk56up9a+1d1tq/E1y8B8Fb6dYJPf+UtfZJgjMAnYCzgOerNe1hQwb/8Oo6fRFHffDBB3Ts2JGbbroJgFatWqnwRWqYytxl7x/AfIL1ei4w1Bhz04HHWGt/IHgd/4XGmN9WR9Aj2X9rXV3nK+IIn8/HuHHj6Nu3L8nJyfzP//yP05FE5Agq2pznVmA08GcAa+0aYCjwD2NM632HhT63heA5/3urLe1hlPuD6wY1vS8Sfhs3bqRXr15MnDiRkSNHsnz5cjp16uR0LBE5gopG+t8AlwBLIXjtfuj6/LeARw9z/HSgvTGmfZWmPIp9C/nKA/boB4pIlfN4PGzdupWZM2cydepUEhMTK/5LIuKYis7pz7PWLiG4cM8QPKcPwRH9b40xp0Nwb/7Q8bsJbuhzabUlPsS+6/PrxHrD9ZYiEa24uJgnnniCQCBAs2bN+O677xgyZIjTsUSkEip7lz1LcJV+AMBamwOcDawHFhGa4g+ZBSyowoyVsu8WuyJSfb799luysrL485//zIcffghAdHS0s6FEpNIq1ZTW2jJr7S3W2rwDnsu21pZYa8+z1pYc8Pz/Wms/q46wh80W+lM78olUH2stzz33HGeddRbbt29n7ty59OnTx+lYInKMav3weN9NPKI8tf5LEamxbrnlFsaMGUP37t3JycnhggsucDqSiByHCvfENMZEAU2stRsrcewpwIPW2iuqItyx0EhfpPpcccUVNGnShNtuuw2PfsEWqbUqsxF2R+ATIGHfE8aYxsA7QLcDp/aBOgRvuxs2/tCqfZW+SNXx+/089NBD5OXl8eCDD9K9e3e6d+/udCwROUGV+ZW9BDh0a91yIB0oO+T5ssMcW618odJX5YtUjS1bttC/f3/uvvtu1q9fTyAQ1ntoiUg1qkzp+0P/HcgHwdvvHvJ82H867NuUZ9/d9kTk+L3zzjukp6fz+eef88ILLzBz5kxN54u4iGvuc6mFfCInZvv27Vx++eW0adOG2bNnc8YZZzgdSUSqWK0v/X2X7GkbXpHjs23bNho1akTDhg2ZO3cuWVlZxMXpVtUiblTZ4XFdY8yaff8BOYA58LnQ8/OrL+rhWd1lT+S4vfzyy5x66qnMmjULgJ49e6rwRVyssiP9EuDvlTiuKXDb8cc5fhrpi1Refn4+N910EzNmzODcc8+lR48eTkcSkTCobOmXWmunV3RQaC/+sJZ+wOqSPZFjsWLFCgYPHsxPP/3E+PHjufvuu4mKqvVn+kSkElzz/3SvUemLVMZPP/1EcXExH3zwAT179nQ6joiE0TGXvjFmNHAuv76MD6DuCSc6TlG64Y7IEe3YsYPFixfzm9/8hiuuuIKLLrpIt8EViUCVKX3DwQv+EoB6hK7VP0Sdqgh1rDSzL3JkH3zwAUOHDqWwsJD169eTkpKiwheJUJUp/bjQfwBYa58AnjjcgcaYM4Cw3WFvH4+m9kV+xefzMX78eB544AFOO+003nnnHVJSUpyOJSIOqrD0rbVfckDpVyAGiD+hRMdBnS9ysPLycvr06cMnn3zC1VdfzRNPPKHRvYhUza11jTEdjTFe4GugUVW85jG+f7jfUqRGi46OZsCAAcycOZMpU6ao8EUEqETpG2O6GmOOeFyo7L8A0gAv0KTq4lWOzumLQHFxMTfeeCMffvghAHfffTdDhgxxNpSI1CiVGenP4ijT+9ZaP8HFfqXAMGB+6BeBsNE5fYl03377LVlZWUyePJklS5Y4HUdEaqjKLOQrA0qNMeNDHx/uTnqW4CV8fwH+E/pFIGxU+hKprLU8//zz/OUvfyEpKYm5c+dywQUXOB1LRGqoypT+vpL/M/AV0ANYDJwNrOaX6/U7AKcAfao4Y4XU+RKp/vvf/zJmzBj69evHjBkzaNy4sdORRKQGO5aFfBboT3Aq/w+hPycBE0KPfw+8Yq3dVdUhK6KRvkSa/Px8AH7zm98we/Zs5s6dq8IXkQodz+p9yy93tD3wuWeBf55wouOghXwSKfx+P/fffz+nnHIKGzZswOPxMGjQIDwe7UgpIhU74vR+aMX+8wR33+tJcGX+/k8f5q/ssNbmVW28ytElexIJtmzZwrBhw/jggw8YMmQIdes6tuu1iNRSRzunH03wVrl1gHcIbrxTI2mkL2739ttvM3LkSIqKipg6dSojR47UL7sicsyOOCdorS211l4IbCBY/LkVvFZbY8wVVRmusvTDT9xu9uzZNG3alOXLlzNq1Cj9mxeR41LZu+zZI/x5oH7ASODVE8x0zHbkl4b7LUWq3erVqwkEApx++ulMnjyZqKgo4uIquyO2iMivVXb1jwn9tyT05/zQ83cDD4YePw/EGGMurNKElZCWFBvutxSpVi+99BIZGRlcf/31ANSpU0eFLyIn7FhG+hNDj6cd8jlDcNV+CfAYcC3w7pFeyBgzBTgTeNtaO/EoxzUC5lprO1cULjZKK5fFHfLz8/njH//ISy+9RM+ePZkxY4bTkUTERSpT+jFAnLX2sJfjmeDJxX8SXN0/A7jXGBNtrS0/zLF/ALzW2nOMMVONMW2stauP8L6PUsk79uk6fXGDtWvX0r9/f9asWcP48eMZO3YsXm9Yd7QWEZerTOk/zS+77h1OHMHRfqy1dqsxps/hCj+kNzAn9Hgewd39flX6xpg+QCGwtRL5tHpfXKFp06acccYZTJkyhZ49ezodR0RcqMJ5cWvtY9baI66Us9YWA62AbaGPvzjKyyUCm0OPd3OY2/AaY2KAccDfjvQixpjrjDHZxphs0Ehfaq8dO3YwZswYcnNziY2N5b///a8KX0SqTZWcDLfWrrfWHm5F/6EK+GXKvs4R3v9vwDPW2r1Heb/nrLWZ1tpMAI+G+lILLVy4kPT0dKZNm8bixYudjiMiESDcK+CWE5zSB0gH1h3mmPOBPxpjPgQ6GWNeqOhF1flSm/h8Pu6++27OP/98kpOTWbp0qe6MJyJhUdnV+1XlDeBjY0xT4EJgsDFmorV27L4DrLX75zaNMR9aa0dX9KKa3pfa5Pbbb+exxx7j6quv5oknniAxMdHpSCISIcJa+tbaPGNMb4Ib+Txsrd0K5Bzl+N6VeV3tTia1QVlZGTExMfz1r3+la9euDBo0yOlIIhJhwn6Bu7V2j7V2Tqjwq4Sm96UmKyoqYsyYMVxyySUEAgGaNWumwhcRR7hiVxtN70tN9c0335CVlcVzzz1HRkYGgUDA6UgiEsHCfU6/WmikLzWNtZZ//etf3HLLLdStW5d58+bRr18/p2OJSIRzxUi/1KfRk9QsBQUFPPDAA/Tq1YucnBwVvojUCK4Y6W/NK3E6gggAy5cvp0OHDiQlJfHpp5/SrFkzPB5X/G4tIi7gip9GpzdKcjqCRDi/38/9999P165deeSRRwBo0aKFCl9EahRXjPS1kE+ctGXLFoYNG8YHH3zAkCFDuPnmm52OJCJyWO4ofQ2mxCELFy5k0KBBFBUVMXXqVEaOHKl9I0SkxnJH6euHrDgkLS2NU089lRdffJG2bds6HUdE5KhcMUbWyErC6YcffuAf//gHAB06dOCzzz5T4YtIreCK0td1+hIuM2bMICMjg0cffZQtW7YA+qVTRGoPl5S+fuhK9crPz2f48OFcddVVdOnShZycHJo2bep0LBGRY+KSc/pOJxA3s9bSp08fVqxYwfjx4xk7dixer9fpWCIix8wVpa/pVakOgUAAYwzGGMaNG0dKSgo9e/as+C+KiNRQLpnedzqBuM327du55JJLeOqppwD47W9/q8IXkVrPJaWv1peqs2DBAtLT01m4cCExMTFOxxERqTKuKP3V2wucjiAuUF5ezl133UW/fv1ITU1l6dKljBkzxulYIiJVxhWlf0aTZKcjiAtkZ2fz4IMPcs0117Bs2TI6duzodCQRkSrlioV8OqcvJ2LVqlWcccYZnHPOOeTk5NChQwenI4mIVAtXjPTV+XI8ioqKGDNmDO3bt2fJkiUAKnwRcTWXjPRV+3JsvvnmGwYPHsy3337LHXfcQUZGhtORRESqnStKX0N9ORYvvPACN998M3Xr1mXevHn069fP6UgiImHhiul9jfTlWOTm5tKrVy9ycnJU+CISUVxR+qp8qcinn37K3LlzAbjlllt45513aNSokcOpRETCyxWlr5G+HInf72fixIn06tWLcePGYa3F4/Hg8bjin76IyDFxxU8+db4czubNmzn//PMZN24cAwcOZMGCBbpPg4hENFcs5NMPcjnU5s2bSU9Pp7i4mBdffJGrrrpK/05EJOK5pPSdTiA1hbUWYwxNmzblpptuYvDgwbRt29bpWCIiNYI7pvedDiA1wg8//EDPnj1ZtWoVxhjGjx+vwhcROYArSl8L+SKbtZbp06eTkZHBypUr2bJli9ORRERqJFeUvjo/cuXn5zN8+HBGjhxJZmYmX331FX379nU6lohIjeSK0tdIP3I99thjzJo1iwkTJrBgwQKaNWvmdCQRkRrLFQv5JLIEAgG2bt1K06ZNueOOOxgwYABZWVlOxxIRqfE00pdaZfv27VxyySV0796dgoICYmNjVfgiIpXkipF+Xkm50xEkDBYsWMCwYcPYs2cPkyZNIjEx0elIIiK1iitG+gFrnY4g1cjn83HXXXfRr18/UlNTWbp0KTfeeKM22xEROUauKP0GdWKdjiDVyBjDZ599xujRo1m2bBkdO3Z0OpKISK3kiul9jffc6bXXXqNbt240btyYuXPnEhcX53QkEZFazRUjfbW+uxQVFXHddddx2WWX8cgjjwCo8EVEqoArRvriHl9//TWDBw9m1apV/O1vf2PChAlORxIRcQ1XlL7RUN8V5s6dy6WXXkrdunV577336Nevn9ORRERcxRXT+1rE7Q5ZWVkMHjyYnJwcFb6ISDVwRelL7fXJJ59w+eWXU1ZWRr169XjxxRdp1KiR07FERFzJFaWvgX7t4/f7ue++++jVqxdffvklmzdvdjqSiIjruaL0pXbZvHkz559/Pvfccw+DBw9mxYoVtGrVyulYIiKu546FfBrq1ypDhgxhxYoVTJs2jREjRmhnPRGRMHFH6WuCv8YrLS3F7/eTkJDAs88+i9fr5fTTT3c6lohIRNH0vlS777//nrPPPpubb74ZgDPPPFOFLyLiAFeUvmaHayZrLdOmTaNLly5s3LiRSy+91OlIIiIRzR2l73QA+ZW8vDyGDRvGqFGjOOuss8jJyeGSSy5xOpaISERzRelLzbNr1y7mzp3Lfffdx/z582nWrJnTkUREIp4rFvJpfr9mCAQCvPHGG1x66aW0atWKn376iZSUFKdjiYhIiCtG+qp8523fvp2LL76Yyy67jLfffhtAhS8iUsO4Y6Qvjpo/fz7Dhw9nz549PPPMM1x88cVORxIRkcNwx0hfQ33HPPzww/Tv35/U1FSWLVvGDTfcoM12RERqKFeUvjinc+fOjB49muzsbDp06OB0HBEROQpXTO9rR77wmjNnDuvXr+e2226jX79+ug2uiEgt4YqRvmaTw6OwsJBrr72WQYMG8eabb+Lz+ZyOJCIixyDspW+MmWKM+dwYM/YIn69rjHnXGDPPGPO6MSYm3Bnl17766isyMzOZMmUKd955Jx988AFRUa6YKBIRiRhhLX1jzB8Ar7X2HKC1MabNYQ4bCkyy1vYHtgIDKnzdqo0ph9izZw89evRg7969vP/++zzwwANER0c7HUtERI5RuIdqvYE5ocfzgB7A6gMPsNY+c8CHacD2Q1/EGHMdcB1ATONTNb1fTYqLi4mPjyc1NZXp06fTvXt3GjZs6HQsERE5TuGe3k8ENoce7wYaHelAY8w5QKq1dvGhn7PWPmetzbTWZlZPTPnkk084/fTTefPNNwG49NJLVfgiIrVcuEu/AIgPPa5zpPc3xtQDngSursyL6rrwquP3+5kwYQK9evUiJiZGe+aLiLhIuEt/OcEpfYB0YN2hB4QW7r0K3GmtXR++aLJp0yb69u3Lvffey5AhQ1ixYgWZmZpMERFxi3CX/hvAcGPMJGAg8K0xZuIhx1wDZAB3G2M+NMYMCnPGiLVw4UKys7OZPn06L7/8MsnJyU5HEhGRKmSsteF9Q2NSgX7AR9barSf6erFN2tjHZr3Ljb1PPfFwEai0tJQVK1ZwzjnnYK3l559/pmnTpk7HEhGRozDGLD+edW1hv07fWrvHWjunKgpfTsz333/P2WefTb9+/dixYwfGGBW+iIiLuWNHPl2pf0ystUybNo0uXbqwceNGZs+eTVpamtOxRESkmrmj9NX5leb3+xk+fDijRo3irLPOIicnh0suucTpWCIiEgauKH2pPK/XS8OGDbnvvvuYP3++LskTEYkgrtg8XQP9owsEAkyaNIlzzz2Xrl27MmnSJKcjiYiIA1wx0tf0/pFt27aNiy66iNtuu41Zs2Y5HUdERBzkipG+HN68efMYMWIEubm5TJ48mTFjxjgdSUREHOSK0tfq/V9bsGABF1xwAWeeeSbz58+nffv2TkcSERGHuWJ6X37h9/sB6N27N48++ijLli1T4YuICOCS0tc5/aBXXnmFM888k61bt+L1evnrX/9KQkKC07FERKSGcEXpR7rCwkJGjx7N4MGDqVevHuXl5U5HEhGRGsgVpb8tr8TpCI756quvyMzMZOrUqdx111189NFHtGjRwulYIiJSA7liIV+TuvFOR3DMAw88QG5uLu+//z59+/Z1Oo6IiNRgrih9T4Sd09+9ezeFhYW0aNGCZ555Br/fr73zRUSkQq6Y3jcRtJLv448/plOnTlx55ZVYa6lXr54KX0REKsUlpe90gurn9/uZMGECvXv3JjY2lscffzyiftkREZET54rpfbfbvn07AwcOZNGiRQwbNoxnnnmGpKQkp2OJiEgt44rSd/t4NzExkaKiIqZPn86IESOcjiMiIrWUK6b33aikpISJEydSWFhIYmIiixcvVuGLiMgJcUfpu+zc9nfffcfZZ5/NuHHjeOuttwDweNzxP5WIiDhHTVKDWGuZOnUqXbp0YfPmzbz11lsMGjTI6VgiIuISrih9t4zzJ06cyDXXXEPXrl3Jycnh4osvdjqSiIi4iCsW8tV21lqMMQwdOpSYmBj+53/+B6/X63QsERFxGXeM9GvpUD8QCPDII48wcOBArLW0bt2aO+64Q4UvIiLVwh2lXwsn+Ldt28ZFF13E7bffTiAQoKQkcm8aJCIi4eGK0q9t5s2bR3p6OosWLeLZZ5/lP//5D/HxkXvTIBERCQ9XnNOvTdP7RUVFjBw5kvr16zN//nzat2/vdCQREYkQrij92mDjxo00bdqUhIQE3nvvPU455RQSEhKcjiUiIhHEFdP7NX2g/8orr9C+fXseeughADp06KDCFxGRsHNF6ddUhYWFjB49msGDB9OuXTuuvPJKpyOJiEgEc0Xp18Rz+l9//TWZmZlMnTqVu+66i0WLFnHyySc7HUtERCKYzulXk+LiYoqKipg/fz59+vRxOo6IiIhLRvo15Kz+rl27mDp1KgBZWVmsXr1ahS8iIjWGK0q/JnT+Rx99RKdOnbjhhhtYt24dADExMc6GEhEROYA7St9BPp+P8ePHc9555xEfH8/nn3+uc/ciIlIjueKcvlMDfWstv/3tb3n33XcZPnw4Tz/9NElJSQ6lEREROTpXlL5TjDFceeWVDBkyhOHDhzsdR0RE5KhcUfomjNfslZSUcNttt5GRkcGoUaMYNmxY2N5bRETkROic/jFYtWoVXbt25amnnuLHH390Oo6IiMgxccdIv5pf31rLiy++yM0330xCQgJvv/02F110UTW/q4iISNXSSL8SsrOzueaaa+jatSs5OTkqfBERqZXcMdKvpqH+jh07SEtL46yzzmLu3Lmcf/75eL3e6nkzERGRauaKkX5Vl34gEODhhx/m5JNPZtmyZQBccMEFKnwREanVXDHSr0rbtm1jxIgRzJs3j8suu4xTTz3V6UgiIiJVwh0j/Spayjdv3jzS09P56KOPePbZZ3n11VdJTU2tktcWERFxmkb6B/j8889p0KABCxYsoF27dk7HERERqVLuGOmfwEB/zZo1fPrppwCMHTuWZcuWqfBFRMSVXFH6x2vWrFl06tSJ0aNH4/f78Xq9xMfHOx1LRESkWkRk6RcWFnL11Vdz5ZVX0qFDB+bOnauV+SIi4noRd05/x44dnHvuufzwww+MHTuWe++9l6ioiPs2iIhIBHJF2x3LDXcaNGjAeeedx+TJkznvvPOqMZWIiEjN4orp/Yoqf9euXQwdOpQ1a9ZgjFHhi4hIRHJF6R/NokWLSE9P59VXX92/u56IiEgkckXpH2523+fzMX78ePr06UNCQgKLFy9m0KBB4Q8nIiJSQ7ii9A9n0qRJ/P3vf2fYsGEsX76cjIwMpyOJiIg4yh0L+Q44q19QUECdOnX44x//yCmnnMJll13mYDIREZGawzUj/ZKSEm666SaysrIoLCwkMTFRhS8iInKAsJe+MWaKMeZzY8zYEznmQBvXrqZr1648/fTTDBgwQNfdi4iIHEZYS98Y8wfAa609B2htjGlzPMccyF+Ux18GD2DLli28/fbbTJo0idjY2Or5AkRERGqxcI/0ewNzQo/nAT2O85j9AsW5tO3YhZycHC666KIqiikiIuI+4Z4HTwQ2hx7vBg63pL7CY4wx1wHXhT4szVn6yTfNmjWr4qhyiAbATqdDuJy+x9VP3+Pqp+9xeJx+PH8p3KVfAOy7jV0dDj/TUOEx1trngOcAjDHZ1trMqo8qB9L3ufrpe1z99D2ufvoeh4cxJvt4/l64p/eX88t0fTqw7jiPERERkWMU7pH+G8DHxpimwIXAYGPMRGvt2KMcc3aYM4qIiLhSWEf61to85ANsVgAACHVJREFUggv1FgPnWWtzDin8wx2TW8HLPlcNUeXX9H2ufvoeVz99j6ufvsfhcVzfZ2OtreogIiIiUgO5Zkc+ERERObpaU/rVsZOfHKyi758xpq4x5l1jzDxjzOvGmJhwZ3SDyv47NcY0MsZ8Ea5cbnIM3+NnjDG/CVcuN6nEz4tUY8w7xphsY8y/wp3PLUI/Bz6u4JhKd1+tKP3q2MlPDlbJ799QYJK1tj+wFRgQzoxucIz/Th/ll8tXpZIq+z02xpwLNLbW/l9YA7pAJb/Hw4F/hy7fSzLG6DK+Y2SMSQWmE9y/5kjHHFP31YrSpxp28pNf6U0F3z9r7TPW2vdDH6YB28MTzVV6U4l/p8aYPkAhwV+u5Nj0poLvsTEmGngeWGeM+V34orlGbyr+d7wLaG+MSQFaABvDE81V/MAgIO8ox/TmGLqvtpT+obv0NTrOY+TIKv39M8acA6RaaxeHI5jLVPh9Dp02GQf8LYy53KQy/5ZHACuBh4EsY8zNYcrmFpX5Hn8CtPz/7d1/rNV1Hcfx54vLFbRMS+ePmo4azV9RwAqstpKU4Y9VOGuNdANZM5ylqFsFm2ltZa2NRaEtKm2tMssKY6UlJeVWWtqiKJq50hlYqy1KAiHi1R+fz7l8ObuXey7ce+mc+3psZ5zz/X7O53z4cLnv8/18vp/3B7gW2FLLxQjY/lcHK9hGFPu6JeiPSia/OKiO+k/Si4BPA0vHqV29ppN+/gBwu+3t49aq3tJJH88C1tr+C/BlYN44ta1XdNLHNwPLbH8Y+D1w5Ti1baIZUezrlsCYTH5jb9j+q1eg3wBW2H5q/JrWUzr5Ob0AuEbSRmCmpM+PT9N6Rid9/ATwsvr81UB+nkemkz5+ITBDUh8wF8j68LExotjXFev0Jb0AeAj4ITWTH/D2ZmKfQcqc28GwSFQd9vHVwEeBTfXQZ2zfPd5t7Wad9HNb+Y22zxu/Fna/Dn+WjwXuoAyF9gNvs711kOpiEB328RzgTsoQ/8+AS23vOALN7Xqt3wOSzgbeeTixryuCPgzcxTgf+EkdkjukMjG09N/4SD+PvfTx2Esf//8Yyb9F1wT9iIiIODzdMqcfERERhylBPyIiYoJI0I+IMSepT5KOdDsiJroE/YgeIWmhpNcNcW7qWO6VIGmBpBsbr2+V9P1GkQ8C6+vyrU7qu7wmgYqIUTT5SDcgIkbNTcCPJK2irItuWQFMAxZJMiWBxzW2PwsgaRbwHMOvo+4DpgK/sb2n7dw/gZWSTrL9fmA3sKvWfzHwPspSo/8231TT4U4Gdtve1zi1FNgJvLlRtg84Cthne/cwbY2IQSToR/QASacDM4C3AHOAebY3SvoiJaAuA5bVshspWbxaHqYE6WbQPYoS4Js5vyfV42dQk9lImgII+DlwCbC6roFvugG42nZrZ8ZJtp+r594BrAF2SWoF8n7KF5C9kp5s1NMPHEPZiOgjHXVMRBwgQT+iNywGHrO9tV7ND2fgitv2lPaTkpYAt9ieNkw9Hweuazs28EWh0ZbzJd1Zn98LLKzP76p/3m/77/U9dwHH1zIzgV/Y3ld3D7uEkgY6Ig5B5vQjupykycC7KFfrLQ/WgLsYmCppg6RnJW2npOwccqvOEboFOAHoty1gOvAM8DgluJ8FfIcyvTCJMnqwqPH+o4FzgcclzZd0D3AKJW/7SuDHwEWSlgKPAqfWOiLiEORKP6L7XUmZp2+aZ3tj64Wk1ZTh+90eJCOXpOuAnbY/N5IPbm4KVLcD/mp9PEvZ1GYH5QvGZuBG22vb3r8DeI+ktcAe4DLKSMEDlC8El9r+br3vYLHtdSNpX0QcKFf6EV2szuV/DLh9iPNTJZ1E2Up2EbBY0hJJr2grOh94Q9uxSZKObzxOlHTqIJ/xqjokvx641fYNlLn/Kbb/XOu+GbhN0vrannb/BrZT7kt4E+U+gSuAl0iaTdmbfXqW/UUcngT9iO62jRJQH2s73hre3wWcBnwCuJwyT74KeHlb+b005vmr04B/NB5/A+5rFpD0GuCXlBvsZtpe3WjXdkl9LlZRNgM5nbbfO5IuAh4BzqeMCtxLuXv/m8ACysY40yirEO6RdMxwnRIRg0vQj+hitvfaXjPIqXl1jv1oSlD+HrDG9kLKMPqjHVT/lG21HpS759vzAGwGzrH9Vtt/aDs3h8aKANsb6rGBlQOSVgJfooxEbKHsxnYs8CnKMsC5wDnAbMoWuK+k3PEfEYcgc/oRPaoOhbeGwzcAF0r6NWXu/umR1md7L2VEoOk+4I0HGXXfN8Q5SZoEfA34uu0n6sG5lP3AvwD80fbyukXr07afkTQTcB1BaB+ZiIhhJOhH9KYHG89fCnyLkhXPwG2j+DkX1zoHkutIOgP4FbAVWG/7+lbhuk6/tURwBmVaYo+k9mQ/z6N8YVjSeC/szxWwgHJnf0SMQIJ+RG9qJefpB/batqQfUObKTxmtD7G9s/la0ouBr1CG7D8E/LSOOKywvatm8ttT37uJIX4HSVoHPGl7+Wi1NSIypx/RK/rY//+5v3XQ9n+A50u6CbgQ2ATcIenkugnOTElnUZb8HSfpTElnUtbD97de18fZtfz09g+XdIKk6ylX+FuAa21vA15LmYvfLOm9ko4buy6IiOHkSj+iN0xlf9KagQx7dQOe+yl3xM+m3IX/SeB3lJviHuHAvPsPt9Xb/rq/1ndZrX85ZSngLOC3wLttf7tVuM7DnwdcRUnks0rS3bavGObv0/wSExGjRIPk6YiIHiLpZNt/bTt2Yivt7WHW/XrgAmBdHa4/WNkplCWD22w/NEzZB4A/2b7qcNsYEfsl6EdEREwQGT6LiIiYIBL0IyIiJogE/YiIiAkiQT8iImKCSNCPiIiYIBL0IyIiJoj/Aa6txHtbxqWXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "def plot_roc_curve(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel('假正类率', fontsize=16)\n",
    "    plt.ylabel('真正类率', fontsize=16)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "plot_roc_curve(fpr, tpr)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9624672915562328"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算曲线下面积AUC，虚线是随机分类0.5到1\n",
    "from sklearn.metrics import roc_auc_score\n",
    "roc_auc_score(y_train_5, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 召回率TPR越高，分类器的假正类FPR就越多\n",
    "* 虚线表示纯随机分类器的ROC曲线，好的分类器应该远离这条线，向左上角\n",
    "* 是使用精度/召回率 PR曲线，还是使用ROC，关键在于 正类非常少或者更关注假正类而不是假负类，选择PR，反之ROC\n",
    "* 例如：前面例子ROC曲线很不错是因为跟负类 非5 相比， 正类 数据5 数量真的很少"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练随机森林分类器，比较SGD分类器的ROC曲线和ROC AUC分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 获取训练集中每个实例的分数\n",
    "* RandomForestClassifier 没有descision_function(),但是拥有dict_proda()方法，sklearn中分类器都有这两个中的一个\n",
    "* dict_proda返回一个矩阵，每行一个实例，每列代表一个类别的概率，比如这个图片 70%是5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\ensemble\\weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# 随机森林分类器\n",
    "forest_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
    "# 交叉验证 交叉执行predict_proba方法 预测概率\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3,\n",
    "                                    method=\"predict_proba\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1. , 0. ],\n",
       "       [0.8, 0.2],\n",
       "       [1. , 0. ],\n",
       "       ...,\n",
       "       [1. , 0. ],\n",
       "       [1. , 0. ],\n",
       "       [1. , 0. ]])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probas_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制ROC曲线，需要决策值不是概率，直接使用正类的概率作为决策值：\n",
    "y_scores_forest = y_probas_forest[:, 1] \n",
    "# 真正  假正 阈值\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_5,y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0.2, 0. , ..., 0. , 0. , 0. ])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_scores_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf0AAAF6CAYAAAATeYHoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVxU1f/H8dcBARFRMXcysyI1UxRxLZc0t9K+mbllbmVqZWVff1aaW2am2dcWK/u6b7n0LcvMMnfLXEFFc8vMchdNRUVkm/P744ADCII6zGUun+fjMQ+Gw+Xe92DNZ865556rtNYIIYQQwv68rA4ghBBCCPeQoi+EEELkE1L0hRBCiHxCir4QQgiRT0jRF0IIIfIJKfpCCCFEPiFFXwghhMgn3F70lVKllVK/ZLPNNKXURqXUUHflEkIIIezOrUVfKRUEzAICrrPNE4C31ro+cJdSKsRd+YQQQgg7c3dPPxnoBFy4zjZNgC9Tni8HHszlTEIIIUS+UMCdB9NaXwBQSl1vswDgWMrzs0BYxg2UUn2APgABAQG1Kleu7NqgHirJoUlKdlz9mujQOBzOZZZ1hicZF2DW6HSN1yzQnP7HZvvs9pnJTvR1tr/2d3T6Y2byC9e+Ln3tz7N9XZn8znVyZvxbZbpfIfKpa97hFai0rTqlTTm/19p87ywPCofDPPP2du4zKcl8LVDAuW1iIjgc4Ouj8PIy+05KgqREs52PT8phNMTFgZcXFPJ3Jr14yWQIDHTu8+JFs31AISiQ8vtXrkD8FUVBfyjoC8oL4uPhciz4+kFAgNljsgPOnzf7LFHC+brPnjU5b7vNZAC4cAES4qFYsZScChKuKK7Em+yFU8bFE+ITOXvyCMmJcRQqEsTlC+fOaK1L5vxfJeXvdqO/4AaXAP+U54XJZDRCaz0ZmAwQHh6uIyIi3JfOjbTWxCYkc/pifJrHFU5fSvN9yvMzlxJIdqQvOwrwtia6IOUNDPMh1yv1TU9x9bmXMj9TqdsqhVLglbEt5bmzXV19c7ymLcPx0u7Dy+vatqyO55XhGKTbr/M5pGZOeU1eaV/ntcfItD2715n2GBl+ltnrzPQ1pfubZ/OaMv4NUp5nlZ102zqPQZp/49S2a/5NMvy7XT2e17X/rmmPoRQphVLh42PaoqPhdDSUKa0oU8b87pkzsC1SUaIE1Klt9puUCHPnQgFvRa+ezmMsWgSrVymGD4PgcgrlBct/ghXLFa1aQauWZrudUfDYY4quXeG9cSZnQgL4+ytCQuD3353/D9x7Lxw4APv2QaVKpq1DB/jqK/jyS/Mc4OOP4ZVX4KWXzHOAHTugZk0IDTXPUwUGwqVLEBMDRYqYtscfh8WL4cPP4PnnTdvYsTB4MAx4DcaNM22//QbVqsF998Hu3c593nmnKfAHD5pCC/Dii/DZZ/DTj1C3rmn79ltYuhSqVjV5lYIjR8y+ypc37WA+hBw5AsWLm2KeKikp5QPMdfu914qNjeWuu+4iwN+HyXNm0KlTJ5RSf9/YXoy8WPQjMUP6m4BQYL+1cVwvIcnBmUwKd2bfxyUm53i/Rf19KBnoR8nCfuZroB+F/QpcfbO4+qaZ8Y00qzfNDG9GmRej7N/Q0heH9Mcgkzc088ab/s0/szfStL9HuuNl8Uaa4bWnZvXKcNyMBflqzszyc21xSN2vsL+kJGeRANNri4+HokXB19e0HTsGf/wBd9wBFSuatuho2LQJSpaE+vVNW3w8TJ1qCs+ECc59vvginDgBkyZB6dKmrUcPmD0b1q+HBx4wbTM/MoVuzBhT7AD2bYV+T0PLltB+mWmLjYWh/4ZCheD1l53HmT8D1q6FXk9BSAXT69y1DSZ/BuVKQbu2ZruLMXDqBKxZBefPpe+1HjgAR4/C7beb7wMDzSOtGjVML7pUKWdb9erQr5/ztQCULWteS+prTvXf/0JyMhQs6GwbMgSefNK8zlT9+kGXLs4PBmA+eJw44ez5p/rrL67x6afmkdbjj5tHWuXLm0daPj5w113X7rPADVbcxMREfHx8CAgIYOLEidSuXZuKqf8R3SRLi75S6j7gKa112ln63wK/KKXKAa2BepaEu0EOh+bs5YQsi7fpjZu285cTc7zfgj5elAoseLWYlwj0pWThgleLeuqjRGFf/ApIv17kP8nJpvcEZuh2+3bzplu7tmk7cgS+/x4uX4aXX3a+4X/+uemhPf+86fkB/O9/phf85JPQrZtpmzMHPvjAFKvp002bw+E8psPh7Lk9+qgpxD//DA0bmrZPP4V334WJE+G558DPD7Ztg3/9yxSpZSnFOCkJ+vc3P3/9dWexW7YM/vwT3nvP2RYba77u3esslOXLm9ectkiWLQtPPGF6zKl8fEyPOvVDSaouXaBePfNaUv9GjzxiinPq3xJMz/v7700xv+025z537TKFOLXgA0RGXvvv9eab17Y1aWIeaZUu7fzwktZTT13bVqeOeaRVrFj6XnZqzjJlrv39vGjXrl107tyZkSNH0qFDBzp27OiS/VpS9LXWTVK+7gGGZvjZBaVUE6A58J7WOsbtAZ1ZuBSflKMe+T+x1w6vZ8XbS1GisO81vXLzPH1BD/D1ll6jsK3du+HcOahVC/xTTupFRJjh3CZN4J57TNtbb5mecZcu0L27aRs9GoYNgxYtTGFUyvSqH3jA9LIOHjTbJSXB0KHmfGrfvs6Ctnix+b3WrZ1F/8AB+O475/cAVao4P0ik8vIyD4fDvIb77zftxYql7/mCKYLh4ebDR6pSpaBt2/TFuGBB8wGkSBFzPji1eH/yifkwk7ZYTZ5sPrQUL+5se+EF80irRg34+uv0bb6+ziH0tPr0ubatbl3n0HaqEiXMh5u0lHL+DcSt0Vrz+eef8+9//5tixYpRPO0/sgvkxeF9tNbncM7gd4u4hGRmbDhE1JHz6Qr6lURHjvcRVMgnXQEvUdjvmh55ycJ+BBXyxctLCrnwfOfPmwJdtqzznG1cnOnV+vjAq686ty1RwrQdPuwsoKmFYu9eSJ2PO3w4/PgjfPihOW8Kpve7bJn5cJAqtWeoNezcaXqgAQGmt5q2t1mxohlCDwlJP7zat2/6gg+mh1+1qtk2VeXK8NNP5rxvWjExJlfaDwNLllz7N8qsGIeFmQ8XaXl7m3PIGbVufW2bi+uAyCPOnj3Lc889x6JFi2jVqhWzZs2iVNrzIC6QJ4u+O2mt+fG3k7yzdC/Hzsdd83N/H29KFcmsR57+cVuAH74FZIFD4XlOnDATnKpWhXLlTNvXX5vzyv/+N7Rvb9o2boQGDaBnT5gxw7Tt3GmGqD/6yJzPrl3bTOp6/XVzXjtt0ff2hpMn4dAhM8ELzFDtvHnpz8+2b5/+vHBqW4kS0Lixs612bfOho0gR5/B62bImZ0bff39tW8Zzs2BypWZLVbiwGU3IqHDha9uEuBUrV67ku+++4/333+fVV1/Fy8v1NSVfF/0Dpy4ycslufv3jHwDuK1uEvo3volwx/6uFPcAvX/+JRB6QeikTmOHk48dNgU3toSYnw/z5ZpuuXZ2/N3OmmUDWv79zaHjsWHNOddkyaN7ctPXoAStWmN55//6m7Y8/YMMGCAoyxdHb20w0A1NAU8+jBwWZnvWiRc6JSwULwqBBZqJYWj/9ZIa9777b2fbFF+aR1rPPmkdaISHpe99getl+fjn6EwqRZyUnJxMVFUVYWBgdO3YkPDycuzKbBegi+bKiXbySyEcrDzBzw18kOTTFCvnwfy0q0aXOHXjLsLu4CVeumEuV/Pycw9xJSWZimFLQubNz2xkzzLnnZ581PVMwveUDB8wM7dQZyB9+aHrKAwaYiWRgJpn16GEmLa1fb4aWk5PNpDMfn/RF/913TaYGDcyELDDfOxxm+Dy16D/yiOmxO9KcyWrRwkwee+QR54S12rXN+etixZxt1arBunXp/xZ+fmbSWUY1atzY31QIuzt27BhPP/00mzZtYv/+/dxxxx25WvAhnxV9h0PzzfZjvPvjPs5cijc9o7p38H8tKhEU4Jv9DoRtJCebouzj45x0deKEmWx1993OWcm7d8PmzaZXXS/lOpLVq6FZMxgxAkaONG0bN0LTpqZ95UrTlpBghq8LFkxf9D/91MxqbtXKWfTLlYPly+Gff5zbpfbuk9NctZk6IzwpyTlM7u1tin3Gy4FeeAHWrEl/OdGgQeYDQtr1rAYMMI+0atY0l0alFRCQ/vy3EOLmff/99/Ts2ZO4uDgmTZpE+YzX/eUWrbVHP2rVqqVzYtfR87rdp+t1hde/1xVe/163+3S93nX0fI5+V1gnOVnr777Tev58raOjne1RUVoPG6b1r78625Ys0bpfP60/+sjZdvmy1uHhWjdokH6/FStqDVr/8YezrXt30/bll+a4Wmv9n/+YtgEDnNt99ZVpq1JFa4fDmadCBa1DQ53bJSRo3bmz2W9aEydqPWSI1ocPO9v+/FPr7du1vnQp/WtP3b8Qwh4cDoceMGCABnSNGjX0vn37bmo/QIS+iZpp+57+udgExi/fz/wth9EaShT2Y3DryrSrGSwz6N1Ia3NO2M/P9GAvXoT//Me0v/668/zvhx+aXvOrr5qe9YkT5jz0hg3m2ueSKYtO/vgjvP22eZ46a1trcxnTww+b67FTRURce+7X19f08tMOad9+u7k8a/du50ph1apBr17prwFu3Bh++cVcUpXaG69e/doFPnx8zLn2jFLPm6eV2XobuTCHRwhhMaUU3t7evPzyy4wbN46CaWexuuP4OrPFzD3I9ZbhXbDlMGOX7eP85UQKeCl6NriTVx4OIbCgT6bbC9dJTDRF2NfXFNagIDP57MoVU4D37zfF+ehRcwlX6shWx47mPPiCBdCpk2kbPhy++QYWLnQOL//4o1nK85574I03TPE9ftx8QGjVynkO2+EwQ+ne3uYyKSGEcDetNbNnzyYkJIQGDRqgtb7l9VeUUpFa6/Ab/j27Fv1fDpym27QtADxwz22MbFuVkNKB12wnbk1yspmAFhBgCveJE+Z88KlT5nxyo0amxxoeborvuXPOVbL69oU9e8xs8KJFTdv69eaDQIMGZtlSIYTwZBcvXuT555/niy++oFu3bsyePdsl+73Zom/bAcRp6w8B8EKTu5n7bF0p+C5w4IBZhnTzZvN9TIyZQFalirkJBZhFQ/r1M8/XrjWT2cBcix0Xl35ZzP/+1wyTpxZ8gAcfNJPepOALITxdREQENWvWZP78+bz99tvMSF3gwkK2LPp/RF9i7f7TFPTx4rmGd8kytjmgdfpZ4osWmSHxtMt1zpplLjObNct8X6SI8/Ky1Dtr+fmZ9cUTE83M9tTTVffem34BFiGEsLMtW7bQoEEDEhISWLduHUOHDsXb2/r7o9iy6M/cYHr57WreLpfiZZCQYM6Hr1/vbHvvPTME/8QTzrbU+1NPm2bOuYPzErNq1cz3SplrxrVOf1ew4OAbv5uUEELYgSNldnCtWrUYOnQoO3bs4MEHH7Q4lZPtin7M5US+jjwGwDMP3GltmDzi/Hnn840bzSS3N95w9uxTr/0+d8653YMPmmu6x41zDrU/+KCZLJd6v2qQGeZCCJFq5cqV1KhRg+PHj+Pt7c3w4cNdfsOcW2W7t+wFWw8Tl5hMw5AS+e48/uHDMH686XmnUsrMnL9yxXxfq5a5cUiBAs5V1cLDza06f/7Z+XvBwWYEoFUrt8UXQgiPlJiYyODBg2nRogVJSUlcuHDB6khZslXRT0p2MHvj3wA880AmFz7byJ9/muvbY1JuPHz5MlSoAK+9ZnrjqVLvDHb0qPlauLBZyW3tWuc2Pj7XrpMuhBAie4cOHaJhw4aMHTuW3r17ExERQeW0S17mMbYq+sv3nOLY+TjuKhFA43tLWh3HpY4dS7/4y1NPmZ74t9+ac++FCjnvM75zp3O7PXvMMH7qfcmFEEK4zqhRo9i3bx8LFy5k8uTJFMrjPShbFf3pKZfp9XzgTo9fbe/iRefzbt3ManHjxzvbunY1q9MdPeo8rz5tmhnaT3v/bX9/Oe8uhBCudPnyZY6mDJ9+8MEHbN++nY4dO1qcKmdsUw52Hj1PxN/nCCxYgPZht2f/C3mYUvDOO86e/bhx5uuvvzrPzb/0EkRHm9ukppIZ80IIkbt27dpFeHg47dq1w+FwUKxYMSpmto52HmWbor9g6xEAOtcuT4Cf51S/xER45RVznh1MT71VK1PoX3jBtJUpY87Z79gh17oLIYQVtNZMmjSJ2rVrc+7cOd599128PHAY1XOqYzb+OHUJgIcqlbI4Sc45HOaWqmfOmMfMmWZS3ejRMHGi8zy8l5cZphdCCOF+MTEx9OrVi2+++YZWrVoxa9YsSpXynFqTlud9TMnC0XOXAShfPO9OonA4zPl2pczkOi8vM0Gvf384eNC5Xa1aMvFOCCHyCl9fX/7++2/ef/99li5d6rEFH2xS9BOSHJy8cAUvBWWK5q3x73PnnEvUxsU515n/8EPnXegmToRNm0wvXwghhPWSk5P56KOPuHjxIv7+/mzevJmBAwd65JB+Wp6dPsXJmCs4NJQt6o+Pd955SbNmmRvQvPIKJCWZO9F17WrO1ffp47wXuxBCiLzj6NGjNGvWjAEDBjBv3jwACthkprQtXkXq0H5wkPUnvo8dM+vTe3lBmzambdky5yp5bduahxBCiLxnyZIl9OrViytXrjBz5ky6py6AYhN5p1t8C46eiwPgdouL/hdfmOvpJ0wwy9redhssWWJ6+TJ0L4QQedunn37KY489xh133EFkZCQ9evSw3V1abVH0j6T09G8PsnYSX+o69YMGOdvatHGucS+EECLvatOmDa+99hobN26kUqVKVsfJFbYo+lb19Ldtg0aNnOvYFy8OP/zgPH8vhBAi79JaM3PmTDp16oTD4aBChQqMGzcOPz8/q6PlGpsU/dSevvuKvsNhhvH/+APatzfL5iplLsmTnr0QQuRtFy5c4Omnn6ZXr15ER0dz6dIlqyO5hU2Kvunpl3fD8P7Jk7Brl5moN3cufPqpudwuMH/dxVcIITzW1q1bCQsLY+HChbz99tusXLmSIkWKWB3LLTx+9r7WuO0afa2hXj2oXRsWLjSFv127XD2kEEIIF0pMTKRjx444HA7WrVvHAw88YHUkt/L4op+Y7EBrKFcs96/RT0qCmjXhq6/MsrkevCiTEELkK6dPnyYoKAgfHx+++eYbKlSoQFBQkNWx3M7jh/cTkh1A7l2jf+IELF9unvv4wNNPw8qVUvCFEMJTrFixgmrVqjFq1CgAatSokS8LPtig6CemFv1iri/6Bw+aG+K0bu28R3379tCsmcsPJYQQwsUSExN54403aNmyJcWLF6dDhw5WR7Kcxxd9h6n5BBZ0/ZmKihVhzx5zHn/MGJfvXgghRC45dOgQDRs2ZNy4cfTu3ZuIiAiqVatmdSzLefw5fYfWKMDfxzXXySUmwvvvQ/Xq8OijUKkSrF8v6+QLIYQnOXfuHIcOHeLLL7+UHn4ant/TT1nU3s9FRf/oURgyBN5803n7Wyn4QgiR98XGxvLFF18AEBYWxqFDh6TgZ+DxRT/1Rjau6ulXrAgNG5rL8lJPHQghhMjbdu7cSXh4ON26dWP37t0AFCpk7dLseZHHF/3Unr6/z82/lPh46NbN+f3PP8OUKXKTHCGEyOu01nz22WfUqVOH8+fPs2LFCqpWrWp1rDzLBkXffPX3vfme/vvvm9X1vvrKRaGEEEK4Rc+ePXnxxRdp2rQpUVFRNJPLq67LFhP5AArewvB+8eJm0Z34eFelEkII4Q4tWrQgNDSUAQMG4OXl8f3YXOfxRT/1nP6tFP3nn4devaBg7q7iK4QQ4hYlJyfzzjvvULZsWZ577jm6du1qdSSP4vEfi5zn9G+s6GttVtpLSjLfS8EXQoi87ejRozRr1owRI0awdetWq+N4JPsU/Rs8p79pE7RsCUWLwoULuZFMCCGEq3z33XeEhoYSERHBrFmzmDx5stWRPJLHD+87bvKSvXr1ICTEzNrPJ3dUFEIIj7R3714ef/xxatSowYIFC7j33nutjuSxPL7o66sT+W5s0EIp+OILCA/PjVRCCCFuVUxMDEWLFqVKlSp88803tGrVCj8/P6tjeTQbDO+brzmdyKe1c/Jf7dqy2p4QQuQ1WmtmzpxJhQoV2LBhAwD/+te/pOC7gA2K/o1N5Bs5EoKDYezYXAwlhBDiply4cIGuXbvSq1cvatasSYUKFayOZCseX/T1DS7Os2wZnDgBR47kYighhBA3bOvWrdSsWZMvv/yS0aNHs3LlSoKDg62OZSsef07/6uI8BXJW9Ddtgs2b4e67czOVEEKIG7VixQqSkpJYt24dDzzwgNVxbMnje/oAvgW88PLK2cl5pczM/ZIlczmUEEKIbJ06dYqNGzcC8PrrrxMVFSUFPxfZoujn5Hz+5cvw2Wey1K4QQuQVy5cvJzQ0lE6dOpGQkIC3tzfFihWzOpat2aLo+xXI/mUsXgwvvgj33eeGQEIIIbKUmJjIG2+8QcuWLbnttttYunQpvr6+VsfKFzz+nD6Aj3f2Rb9lS6hTB6pXd0MgIYQQmbpw4QItWrRg8+bN9OnThw8++EDue+9Gtij6ObmxUvHiZua+3IRJCCGsExgYyP3338/AgQPp0KGD1XHyHVuUQO8crrATFGTW2hdCCOE+sbGxvPjiixw4cAClFFOnTpWCbxF79PSzKforV8KZM1C/Psg6D0II4T47d+6kU6dO7N+/n+rVqxMSEmJ1pHzNFj397C7XGzoUunSBlKtChBBC5DKtNZ9++il16tQhJiaGlStX0rdvX6tj5XtuL/pKqWlKqY1KqaFZ/DxIKfWDUipCKfXfnOwzu+H9Z54xRb9JkxvPK4QQ4sb997//pX///jRr1oyoqCiaNm1qdSSBm4f3lVJPAN5a6/pKqelKqRCt9YEMm3UDvtBaf6GUmqeUCtdaR1xvv9n19Pv0gV69wMfn1vILIYS4vvj4ePz8/OjRowe+vr706tULJXc2yzPc3dNvAnyZ8nw58GAm2/wD3K+UKgaUB7JdJf96V+ylrs0vBV8IIXJPcnIyb731FjVq1ODixYv4+/vzzDPPSMHPY9xd9AOAYynPzwKlM9lmPVABeBnYm7JdOkqpPinD/xGQ9fC+1jBqFEyfDjExLkgvhBDiGkePHqVp06aMHDmS8PBwq+OI63B30b8E+Kc8L5zF8UcA/bTWo4B9QK+MG2itJ2utw7XW4ZD18P4vv5hb6f7f/4Gs/SCEEK63ePFiQkNDiYyMZNasWcyZM4fAwECrY4ksuPuSvUjMkP4mIBTYn8k2QUA1pdQmoC6wMrudZtXTv/tumDEDYmNleF8IIVzN4XAwfvx4KlSowIIFC7j33nutjiSy4e6i/y3wi1KqHNAa6KyUGq21TjuT/11gBmaIfyMwP7udZtXTDw6Gnj1vNbIQQoi09u3bx2233UbJkiVZtGgRRYsWxc/Pz+pYIgfcOryvtb6Amcy3CXhIax2VoeCjtd6ita6qtS6stW6utb6U3X5zuiKfEEKIm6e1Zvr06dSqVYt///vfAJQqVUoKvgdx+3X6WutzWusvtdYnXbVP70x6+klJ0L49TJ3qqqMIIUT+FRMTw1NPPcWzzz5L3bp1GTdunNWRxE2wxYp8mXX09+2DRYvguefcn0cIIexk9+7dhIWF8b///Y/Ro0ezYsUKypUrZ3UscRNssfZ+Zj19Pz8YN04m8AkhxK0qVaoUpUuXZs6cOTRo0MDqOOIW2KPoZ9LVDwmB116zIIwQQtjAqVOn+M9//sOYMWMoWbIkv/76qyy0YwO2GN7PbPZ+TIxzNT4hhBA5t3z5cqpXr87EiRPZtm0bgBR8m7BF0c/Y09caxo6FH38Eh8OiUEII4WESEhJ47bXXaNmyJSVLlmTr1q3UqVPH6ljChexR9DP09P/6yxT9Rx8FL1u8QiGEyH29evVi/Pjx9O3bly1btnD//fdbHUm4mC3O6Wcc3lcKBg6UXr4QQuREcnIy3t7eDBw4kCeeeIL27dtbHUnkElsUfe8Mp5ruvBPef9+SKEII4TFiY2N5+eWX8fX1ZdKkSYSFhREWFmZ1LJGLbDH4nbGnL5P4hBDi+qKioggPD2fGjBkUL14cLW+a+YItin7aiXwOB0yYAD//LIVfCCEy0lrzySefULduXWJiYli5ciXvvPOOzM7PJ+xR9NP09KOiYNQoaNIk85X6hBAiPzt27BiDBw+mWbNmREVF0bRpU6sjCTeyxTn9tJ9QCxeGvn3h7FkLAwkhRB7z22+/UbVqVW6//Xa2bNlC5cqVpXefD9mkp+98HhICEyfCnDnW5RFCiLwiKSmJkSNHEhoayty5cwGoUqWKFPx8yhY9fa8M//HKevtCCAFHjhyha9eu/PLLL3Tr1o3HH3/c6kjCYrYo+mlL/o4d4OsLd99tbrojhBD50dKlS+nevTvx8fHMnj2bbt26WR1J5AG2GN5PO0z11FNQtSrs329hICGEyAMqVqzI9u3bpeCLq2xR9NO65x6oVAmCg61OIoQQ7rV3715mzJgBwKOPPsrmzZsJCQmxOJXIS2wxvJ/2nP5331kYRAghLKC1Zvr06bz88ssULVqUDh06ULhwYby9va2OJvIYW/T0ZRKqECK/iomJoUuXLvTu3Zt69eoRERFB4cKFrY4l8ihb9PRTa35MjPkAEBgoHwSEEPYXHx9PnTp1OHjwIGPGjOG1116T3r24Llv19P/1LyhaFD791No8QgiRm1LXyffz8+PVV1/l559/ZvDgwVLwRbZsUfRTz+mnFn+5VE8IYVcnT56kdevW/PjjjwD069ePBg0aWJxKeApbFP3U8f01a2DPHujSxdo4QgiRG5YvX05oaCjr1q3jzJkzVscRHsgWRV+lWZ6nShWz/r4QQthFQkICr732Gi1btqRkyZJERETItffiptii6HvJpD0hhI0tXryY8ePH069fP7Zu3UrVqlWtjiQ8lC2KvlJmBb6yZaF1a6vTCCGEaxw+fBiAJ598kl9//ZVJkybh7+9vcSrhyexR9FFER8PJk+ayPSGE8GSxsbE888wzVK1alUOHDttL1CoAACAASURBVKGUksl6wiXscZ2+gnr14PhxSEqyOo0QQty8HTt20LlzZ37//XeGDBlC+fLlrY4kbMQmRV/h42OG94UQwlN98sknDBw4kNtuu42VK1fStGlTqyMJm7HJ8L4QQni+Xbt20bx5c6KioqTgi1xhk54+DB4Mhw7Biy9Cw4ZWJxJCiJxZt24dRYoUoWbNmkycOBEfH590twsXwpVs0tNX/P47LFwIkZFWpxFCiOwlJSUxYsQImjZtytChQwHw9fWVgi9ylS16+l4KBg6E+vXN+vtCCJGXHTlyhK5du/LLL7/QvXt3PvnkE6sjiXzCFkVfKWjQwDyEECIv++2332jUqBGJiYnMmTOHp59+2upIIh+xx/C+DIcJITxE5cqV6dSpE9u2bZOCL9zOJkUfRo2CefMgPt7qNEIIkd7evXtp3bo1Z86coUCBAkyaNImQkBCrY4l8yBZFH60YMQK6drU6iBBCOGmtmTZtGuHh4URGRnLw4EGrI4l8zhZFX2t44QWzKp+fn9VphBACYmJi6NKlC71796Z+/fpERUVRt25dq2OJfM4WE/kKeMOnn1qdQgghnAYNGsRXX33FmDFjeP311/HyskUfS3g4WxR9L5nIJ4TIAxwOBzExMQQFBfHOO+/Qq1cv6tevb3UsIa6yRdFPSoKDB6FwYShd2uo0Qoj86OTJk3Tv3p3Lly+zdu1aSpYsScmSJa2OJUQ6thhvOnkS7rkHHnnE6iRCiPzop59+IjQ09OpiO97e3lZHEiJTtij6Xl6Ku+6C22+3OokQIj9JSEhg0KBBtGrVilKlShEREUGfPn1k7RCRZ9lieP+O8mZ4Xwgh3Ck+Pp5vv/2Wfv36MWHCBPz9/a2OJMR12aLoy2dqIYQ7LV68mBYtWhAYGEhkZCRFihSxOpIQOWKL4f3z5xWnT1udQghhd5cuXaJXr148/vjjfJpynbAUfOFJbFH0t22DatXglVesTiKEsKsdO3ZQq1YtZs2axbBhwxgwYIDVkYS4YbYY3o+9pDh1CsqUsTqJEMKOFi5cSPfu3SlRogSrVq3ioYcesjqSEDfFFj39Vq3g0CHo29fqJEIIOwoLC6Ndu3ZERUVJwRcezRZFv6A/3HknFC9udRIhhF2sW7eOl19+Ga01ISEhLFiwgBIlSlgdS4hbYouiD+amO0IIcauSkpIYMWIETZs2ZdmyZfzzzz9WRxLCZWxR9Od/Ac89B7t2WZ1ECOHJjhw5wkMPPcSoUaPo1q0b27Ztk969sBVbFP3o0zBtGvz+u9VJhBCeKjk5mYcffpgdO3Ywd+5cZs6cSeHCha2OJYRL2WL2/sMPQ6+GUK+e1UmEEJ7mypUr+Pj44O3tzeTJkwkODuaee+6xOpYQucIWPf1q1eDFFyE42OokQghPsnfvXurUqcP48eMBaNy4sRR8YWu2KPpCCHEjtNZMnTqVWrVqcfLkSUJDQ62OJIRbuL3oK6WmKaU2KqWGZrPdZ0qptjnZ55Lv4IcfIDHRNRmFEPYVExNDly5deO6552jQoAFRUVG0bt3a6lhCuIVbi75S6gnAW2tdH7hLKRWSxXYNgTJa6yU52e/y5fDoo5CQ4MKwQghb2rNnD99++y1jxoxh+fLllC1b1upIQriNuyfyNQG+THm+HHgQOJB2A6WUDzAF+EEp9S+t9eLsdhoaqihcBuSulkKIzDgcDtauXUvTpk2pX78+f/31F2Vk3W6RD7l7eD8AOJby/CxQOpNtugN7gPeAOkqplzJuoJTqo5SKUEpFgLlGf8kS8JIZCkKIDE6cOEGLFi1o1qwZkZGRAFLwRb7l7jJ5CUjtjxfO4vg1gcla65PAXOCaha611pO11uFa6/BcSyqE8HjLli0jNDSUDRs2MGXKFMLCwqyOJISl3F30IzFD+gChwF+ZbPMHcFfK83Dg7+x2mhAPycmuiCeEsIuhQ4fSunVrypQpQ0REBL1790YpZXUsISx1y0VfKeWVMvEuJ74FuimlJgAdgd1KqdEZtpkGPKSU+hl4AXg/u52+9DLcffeNpBZC2F358uV54YUX2Lx5M/fdd5/VcYTIE5TO5k41SilfYCAwFiiotY5LaS8IdMJMzPtHa10oRwdUKghoDvycMoR/S/zKhmjf0qsoeeEO/vzzVvcmhPBk8+bNw9vbm06dOlkdRYhcpZSKvJlT3Dnp6XsBg4CXgOFp2ucCQwAF5PgKea31Oa31l64o+Kk+/xx++slVexNCeJpLly7Rs2dPunbtyqxZs8iuMyNEfpWTS/YSgFjgByBCKbURCMFcfldLa31ZKWXpGXVvLwjJ9Ip/IYTdbd++nc6dO3PgwAGGDRvG8OHD5dy9EFnItuhrrR1KqUSt9R9KqVeBw8B2YAvwL6XUl9ffgxBC5I4///yTevXqUbJkSVavXk2TJk2sjiREnnajE/lOaq13ACWAj4HxQHmXp7pBY96FkSOtTiGEcJekpCQA7rrrLj766CN27NghBV+IHMhx0VdK1QG+Vkq1wlxK9ydwSmu9FXNe3zKH/kQm8QmRT6xdu5Z7772X7du3A9CvXz9KlChhcSohPMN1i75Sqp5SKnUZ3O2Ynv23mNX0OgBBKZff+SulJqQ8PlRKfZ6rqTMYPBiGDHHnEYUQ7paUlMTw4cNp2rQpPj4+eMkSnELcsOzO6d+FWTrXB/gGGAm8grmWXgMXgLsxHx4qpvyON1AwF7Jm6e67oXJldx5RCOFOhw8fpmvXrqxfv56ePXsyceJEChcubHUsITzOdYu+1noeME8pdRRT4Mdhin0zYDFQCHgWOKC1bpfLWbOkrD27IITIZTNmzCAqKoq5c+fStWtXq+MI4bFyOj6WoLV+CjgHFAWuAE8CRYAKmA8CllnyPaxZY2UCIYSrxcXFsWfPHgCGDBnCzp07peALcYtu9KTY50AV4B/M0H+41jrS5alu0DffwIoVVqcQQrjKnj17qFu3Li1atCAuLg4fHx/uvPNOq2MJ4fGyLfrKrHLhp5QqDizAnN8PwFyyVyp34+VM2zYgV+sI4fm01kyePJnw8HBOnTrF1KlT8ff3z/4XhRA5kpMV+fww5+5bAfO11r8BKKW6A7OVUg0A39yLmL12T0ALucmuEB4tLi6OHj168L///Y+HH36YOXPmyH3vhXCxnAzvJwH9Mb38N1IbtdY/Ah8CDswHAyGEuGl+fn7Ex8czduxYfvrpJyn4QuSCnCzDmwR8kfJtbIafvZsy/F8rF7Ll2OG/4XQFKFnSyhRCiBvlcDiYMGECHTt25I477uDbb7+VdfOFyEW3vLqFNna6IszNGjkS5s2zMoEQ4kadOHGCFi1aMGjQIGbNmgUgBV+IXJajoq+U8lNKLVJK+aV8X0IpVUopFaCUSlZKBaTZdrZS6oHcCpyZO+6AUnliSqEQIid+/PFHQkND2bBhA1OmTGHo0KFWRxIiX8huGd7Uj90O4F8pXwGmAz8BiZh19+NTti8CdAbK5UbYrLz1FnTp4s4jCiFu1oIFC3jkkUcoU6YMERER9O7dW3r4QrhJdj39xUqpx7TWiQBa60Sl1HOYmfwDtdYJplknpWzfHbOAz7e5ljgT8oYhRN6ntVnD69FHH2X48OFs3ryZ++67z+JUQuQvWRZ9pZQX5iY781Muz0MpVR74D/Ca1np1hu0LAgOAEakfEtxFW7oeoBAiO3PnzuXBBx8kLi6OwMBA3nrrLbn+XggLZFn0tdYOrfUIzN30uqU0fwxs1lp/mMmvvAucACa7PGU2+veHRYvcfVQhRHYuXbpEz5496datG15eXly8eNHqSELkazm5ZO8H4AellAN4HbgE5ny/NuN1Sin1H+BxoJ7W2pH13nJHbCycOOHuowohrmf79u107tyZP/74g+HDhzNs2DAKFMjJemBCiNxy3f8DlVLLgMsp32pgLOCVMov/vFKqTsrP2gL1tdanci3pdXz8EXSpb8WRhRCZ0VrzwgsvEBsby+rVq2ncuLHVkYQQZN/T30bKzHxMT74KsBCz7O5xYAPwEXA7MFwp9Yq7z+cDFC0KcnpQCOudOXOGAgUKUKxYMebNm0dgYCAlSpSwOpYQIsV1Z+9rrYdord/CTN4Dcyvdwintn2itJ2JGAGoAtYEpuZpWCJFnrVmzhurVq9O/f38AKlasKAVfiDwmJ3fZexdYiSnuDYGuSqn+abfRWv+OuY6/tVLqsdwIej3Tp8Peve4+qhACICkpiWHDhtGsWTOKFCnC//3f/1kdSQiRhewW5/k30Bt4BUBr/SfQFXhXKXVX6mYpPzuOOec/ItfSZmHdOjh50t1HFUIcOXKExo0bM3r0aHr27ElkZCQ1atSwOpYQIgvZ9fR/A9oAW8Bcu59yff73wPuZbD8LuF8pdb9LU2bjmWegUiV3HlEIAeDl5cXJkyeZN28e06dPJyAgIPtfEkJYJrtz+su11psxE/cU5pw+mB79Y0qpSmDW5k/Z/ixmQZ92uZY4Ew89BOXcuvCvEPlXXFwcH3/8MQ6Hg+DgYPbt20cXWQdbCI+Q07vsacwsfQeA1joKqAf8DawjZYg/xXxglQszCiHyiN27d1OnTh1eeeUV1q5dC4CPj4+1oYQQOZajoq+1TtBav6q1vpCmLUJrfUVr/ZDW+kqa9o+01htyI2xWIiMgJsadRxQif9FaM3nyZGrXrk10dDTLli2jadOmVscSQtygnPb087QPP4QjR6xOIYR9vfrqq/Tt25cHHniAqKgoWrZsaXUkIcRNyHZNTKVUAaCs1jrbsqqUuhsYq7Xu4IpwORVWS1G0qDuPKET+0qFDB8qWLcugQYPw8rJFX0GIfEnpbG5Rp5QKA9ZrrQulaSsD/AA0SDu0r5QKTdk2MJfyXsOvbIhe+MNaHq8Z7K5DCmF7ycnJjBs3jgsXLjB27Fir4wghMlBKRWqtw2/093Lykf0KkHFp3UQgFEjI0J6QybZCCA9y/PhxWrRowZtvvsnff/+Nw+H2e2gJIXJJTop+csojrSQwt9/N0G7Ju0NSEmQzYCGEyIEffviB0NBQNm7cyNSpU5k3b54M5wthI7b4v7lnT4iOtjqFEJ4tOjqaJ598knLlyhEZGcmzzz6LUir7XxRCeAxb3Nzaywu8va1OIYRnOnXqFKVLl6ZUqVIsW7aMOnXqULBgQatjCSFyQU57+kWVUn+mPoAoQKVtS2lfmXtRszZ7NsjNvIS4cXPnzuWee+5h/vz5ADRq1EgKvhA2ltOe/hXgrRxsVw4YdPNxhBDucPHiRfr378/s2bNp2LAhDz74oNWRhBBukNOiH6+1npXdRilr8bu96MskPiFybtu2bXTu3JmDBw8ycuRI3nzzTQoUsMWZPiFENmzxf/qYMfDYQjmvL0ROHDx4kLi4ONasWUOjRo2sjiOEcKMbLvpKqd5AQ669jA/AknXx9uyRgi/E9Zw+fZpNmzbRtm1bOnTowCOPPCK3wRUiH8pJ0Vekn/BXCChOyrX6GRR2RagbNXiwFUcVwjOsWbOGrl27Ehsby99//02xYsWk4AuRT+Wk6BdMeQCgtf4Y+DizDZVSVQC33mEPoFo1dx9RiLwvKSmJkSNHMmbMGO69915++OEHihUrZnUsIYSFsi36WusdpCn62fAF/G8pkRDiliUmJtK0aVPWr1/PM888w8cffyy9eyGEa1bkU0pVV0p5A7uA0q7Y541YvtzdRxQib/Px8aFVq1bMmzePadOmScEXQgA5u8teXWBrJuvsp/7cG3OjnWDgH+BurfU+VwfNil/ZEF2o9FrO7ZC77In8LS4ujoEDB9KxY0eaNGlidRwhRC7Kzbvszec6w/ta62TMZL944GlgZcoHAbdp3tydRxMi79m9ezd16tRh0qRJbN682eo4Qog8KicT+RKAeKXUyJTvM+vxa8wlfAOAr1I+CLhN167uPJoQeYfWmilTpjBgwAACAwNZtmwZLVu2tDqWECKPyknRTy3yrwA7gQeBTUA94ADO6/WrAXcDTV2cUQiRhe+++46+ffvSvHlzZs+eTZkyZayOJITIw25kIp8GWmCG8p9I+ToBGJXy/HFgodb6H1eHzM7p0+4+ohDWunjxIgBt27ZlwYIFLFu2TAq+ECJbNzN7X6c8MrZ9DvznlhPdhKFvWnFUIdwvOTmZd955h7vvvpvDhw/j5eVFp06d8PJyyYU4Qgiby3J4XynlBUzBrL7XCEg7OU9l8iuntdYXXBsvZ0qWtOKoQrjX8ePHefrpp1mzZg1dunShaFFLVr0WQniw653T98HcKrcw8ANm4Z08afQ7VicQInctXbqUnj17cvnyZaZPn07Pnj1RKrPP3kIIkbUsxwS11vFa69bAYUzhj8lmX5WVUh1cGU4IYSxYsIBy5coRGRlJr169pOALIW5KTu+yp7P4mlZzoCfwv1vMJIQADhw4gMPhoFKlSkyaNIkCBQpQsGBOV8QWQohr5XT2j0p5bE75ujKl/U1gbMrzKYCvUqq1SxPmwDsyvC9sZs6cOYSFhdGvXz8AChcuLAVfCHHLbqSnPzrl+cwMP1OYWftXgA+A54Afs9qRUmoacB+wVGs9+jrblQaWaa1rZhcuOjq7LYTwDBcvXuTFF19kzpw5NGrUiNmzZ1sdSQhhIzkp+r5AQa11ppfjKXNy8T+Y2f2zgRFKKR+tdWIm2z4BeGut6yulpiulQrTWB7I47vvk8I59Q4bkZCsh8rZDhw7RokUL/vzzT0aOHMnQoUPx9nbritZCCJvLSdH/FOeqe5kpiOnt+2mtTyqlmmZW8FM0Ab5Meb4cs7rfNUVfKdUUiAVO5iAfpd1+Xz8hXK9cuXJUqVKFadOm0ahRI6vjCCFsKNtz+lrrD7TW8df5eRxQETiV8v326+wuADiW8vwsmdyGVynlCwwD3shqJ0qpPkqpCKVURHb5hcjLTp8+Td++fYmJicHPz4/vvvtOCr4QIte4ZBkvrfXfOrt79BqXcA7ZF87i+G8An2mtz1/neJO11uGptxVculQuXxKeZ/Xq1YSGhjJz5kw2bdpkdRwhRD7g7rU7IzFD+gChwF+ZbPMw8KJSai1QQyk1NbudrlvnqnhC5L6kpCTefPNNHn74YYoUKcKWLVvkznhCCLfI6ex9V/kW+EUpVQ5oDXRWSo3WWg9N3UBrfXVsUym1VmvdO7udPvpIrmQVIle89tprfPDBBzzzzDN8/PHHBAQEWB1JCJFPqJyNyrvwgEoFYRby+VlrnaOJetfjVzZE/2/ZOh4LLXfr4YTIRQkJCfj6+nLs2DHWr19Pp06drI4khPBQSqnI1FPcN8Ltt+bSWp/TWn/pioIvhCe4fPkyffv2pU2bNjgcDoKDg6XgCyEsYYv7cf71l9UJhMjcb7/9Rp06dZg8eTJhYWE4HA6rIwkh8jFbFP2pU6xOIER6Wms+//xzateuzZkzZ1i+fDljx46lQAF3T6MRQggnWxT922+3OoEQ6V26dIkxY8bQuHFjoqKiaN68udWRhBDC7bP3c0WbNlYnEMKIjIykWrVqBAYG8uuvvxIcHIyXly0+WwshbMAW70a3l7c6gcjvkpOTeeedd6hbty7jx48HoHz58lLwhRB5ii16+rIen7DS8ePHefrpp1mzZg1dunThpZdesjqSEEJkyhbdkPfGWZ1A5FepS+lu3ryZ6dOn88UXX1CkSBGrYwkhRKZs0dNPut49AIXIRSVLluSee+5hxowZVK5c2eo4QghxXbbo6Q8aZHUCkZ/8/vvvvPvuuwBUq1aNDRs2SMEXQngEWxR9Hx+rE4j8Yvbs2YSFhfH+++9z/PhxAJSSWSVCCM9gi6IvRG67ePEi3bp1o0ePHtSqVYuoqCjKlZP7PQghPIstiv6SJVYnEHamtaZp06bMmzePkSNHsnr1am6XFaGEEB7IFhP59u+3OoGwI4fDgVIKpRTDhg2jWLFiNGrUKPtfFEKIPMoWPf22ba1OIOwmOjqaNm3a8MknnwDw2GOPScEXQng8WxT9SpWsTiDsZNWqVYSGhrJ69Wp8fX2tjiOEEC5ji6Ivk6eFKyQmJjJkyBCaN29OUFAQW7ZsoW/fvlbHEkIIl7FF0d+31+oEwg4iIiIYO3Yszz77LFu3bqV69epWRxJCCJeyRdFfscLqBMKT7d1rPjXWr1+fqKgopkyZQkBAgMWphBDC9WxR9CvJYmjiJly+fJm+ffty//33s3nzZsCssCeEEHZli0v2WrSwOoHwNL/99hudO3dm9+7dvP7664SFhVkdSQghcp0tir4QN2Lq1Km89NJLFC1alOXLl9O8eXOrIwkhhFvYYng//orVCYQniYmJoXHjxkRFRUnBF0LkK0prbXWGW+JXNkQ//Pg6lk6SddBF1n799VcuXrxIq1atcDgcAHh52eIzrxAiH1JKRWqtw2/092zxrlfQz+oEIq9KTk5m9OjRNG7cmGHDhqG1xsvLSwq+ECJfssU7X69nrE4g8qJjx47x8MMPM2zYMDp27MiqVavkNrhCiHzNFhP5FPJGLtI7duwYoaGhxMXFMWPGDHr06CEFXwiR79mi6AuRSmuNUopy5crRv39/OnfuTOXKspCDEEKATYb3l3xvdQKRF/z+++80atSIvXv3opRi5MiRUvCFECINWxT96FNWJxBW0loza9YswsLC2LNnD8ePH7c6khBC5Em2KPpt2lidQFjl4sWLdOvWjZ49exIeHs7OnTtp1qyZ1bGEECJPskXRL13a6gTCKh988AHz589n1KhRrFq1iuDgYKsjCSFEniUT+YTHcTgcnDx5knLlyvH666/TqlUr6tSpY3UsIYTI82zR09/1m9UJhLtER0fTpk0bHnjgAS5duoSfn58UfCGEyCFbFP1ffrY6gXCHVatWERoayurVqxk0aJDc814IIW6QLYp+1apWJxC5KSkpiSFDhtC8eXOCgoLYsmULL7zwgiy2I4QQN8gWRb9xY6sTiNyklGLDhg307t2brVu3Ur16dasjCSGER7LFRD7p8NnTokWLaNCgAWXKlGHZsmUULFjQ6khCCOHRbNHTP3fO6gTClS5fvkyfPn1o374948ePB5CCL4QQLmCLov/DD1YnEK6ya9cuateuzdSpU3njjTcYO3as1ZGEEMI2bDG8X6yY1QmEKyxbtox27dpRtGhRfvrpJ5o3b251JCGEsBVb9PQffdTqBMIV6tSpQ+fOnYmKipKCL4QQucAWRV94rvXr1/Pkk0+SkJBA8eLFmTFjBqVlXWUhhMgVUvSFJZKTk3n77bdp3LgxO3bs4NixY1ZHEkII27NF0V+82OoE4kYcO3aMhx9+mOHDh9O5c2e2bdtGxYoVrY4lhBC2Z4uJfJcvW51A3IguXbqwbds2Zs6cSffu3WVlPSGEcBNbFP22ba1OILITHx9PcnIyhQoV4vPPP8fb25tKlSpZHUsIIfIVWwzvFy5sdQJxPfv376devXq89NJLANx3331S8IUQwgK2KPoib9JaM3PmTGrVqsWRI0do166d1ZGEECJfs0XRj4y0OoHI6MKFCzz99NP06tWL2rVrExUVRZs2bayOJYQQ+Zotiv4ff1idQGT0zz//sGzZMt5++21WrlxJcHCw1ZGEECLfs8VEvrAwqxMIAIfDwbfffku7du2oWLEiBw8epJiskSyEEHmGLXr6ISFWJxDR0dE8+uijtG/fnqVLlwJIwRdCiDzGFj19Ya2VK1fSrVs3zp07x2effcajcjMEIYTIk2zR0z950uoE+dd7771HixYtCAoKYuvWrTz//POy2I4QQuRRtij6v/xidYL8q2bNmvTu3ZuIiAiqVatmdRwhhBDXYYvhfbkpm3t9+eWX/P333wwaNIjmzZvLbXCFEMJD2KKn36iR1Qnyh9jYWJ577jk6derE4sWLSUpKsjqSEEKIG+D2oq+UmqaU2qiUGprFz4sqpX5USi1XSn2jlPJ1d0ZxrZ07dxIeHs60adMYPHgwa9asoUABWwwUCSFEvuHWoq+UegLw1lrXB+5SSmV2sV1XYILWugVwEmjlzoziWufOnePBBx/k/PnzrFixgjFjxuDj42N1LCGEEDfI3T39JsCXKc+XAw9m3EBr/ZnWekXKtyWB6IzbKKX6KKUilFIRAN98kzth87u4uDgAgoKCmDVrFlFRUTRr1sziVEIIIW6Wu4t+AHAs5flZIMspeEqp+kCQ1npTxp9prSdrrcO11uEADkduRM3f1q9fT6VKlVi8eDEA7dq1o1SpUhanEkIIcSvcXfQvAf4pzwtndXylVHFgIvBMTnb6+OMuySaA5ORkRo0aRePGjfH19ZU184UQwkbcXfQjcQ7phwJ/ZdwgZeLe/4DBWuu/c7JTb29Xxcvfjh49SrNmzRgxYgRdunRh27ZthIeHWx1LCCGEi7i76H8LdFNKTQA6AruVUqMzbPMsEAa8qZRaq5Tq5OaM+dbq1auJiIhg1qxZzJ07lyJFilgdSQghhAsprbV7D6hUENAc+FlrfcsL6PqVDdFvvP0zb/Uue+vh8qH4+Hi2bdtG/fr10Vpz4sQJypUrZ3UsIYQQ16GUikyd13Yj3H6dvtb6nNb6S1cU/FQnTrhqT/nL/v37qVevHs2bN+f06dMopaTgCyGEjdliRb46daxO4Fm01sycOZNatWpx5MgRFixYQMmSJa2OJYQQIpfZouhL5zTnkpOT6datG7169aJ27dpERUXRpk0bq2MJIYRwA1lHNZ/x9vamVKlSvP322wwePBhvufRBiGtcuHCB6OhoEhMTrY4i8hkfHx9KlSqVaxOpbVH0Dx8G5K6uWXI4HEyYMIGGDRtSt25dJkyYYHUkIfKsCxcucOrUKYKDg/H390cpZXUkkU9orYmLi+PYMbOGXW4UflsM7+/YYXWCvOvUqVM88sgjDBo0iPnz51sdR4g8Lzo6muDgYAoVKiQFX7iVUopChQoRHBxMdPQ1K9C7hC16+uXLW50gb1q+fDndu3cnJiaGSZMm0bdvX6sjtaTuBAAAG2VJREFUCZHnJSYm4u/vn/2GQuQSf3//XDu1ZIuiX7Om1QnynlWrVtGyZUvuu+8+Vq5cyf333291JCE8hvTwhZVy878/WwzvC6fk5GQAmjRpwvvvv8/WrVul4AshhABsUvQTEqxOkDcsXLiQ++67j5MnT+Lt7c3AgQMpVKiQ1bGEEBY7c+YMTz31FEFBQZQqVYphw4Zd/dmVK1fo168fRYsWpXTp0owZM+bqz0aOHIlSCi8vL0qVKkXHjh3Zv3+/FS9BuIgtiv7y5VYnsFZsbCy9e/emc+fOFC9eXC4zEkKk06lTJ44fP87XX3/N4MGDeffdd1m4cCEAL7/8MkuXLmXu3LmMGjWKt956i6+//vrq75YtW5ZNmzbx4YcfsnPnTho0aMDhw4eteiniVmmtPfrhW+Ye3W3IcZ1fRUVF6cqVK2ullB4yZIhOSEiwOpIQHm3Pnj1WR3CpQ4cOaUBv27btalu7du30I488oo8fP669vb31/Pnzr/6sR48eunHjxlprrUeMGKErVKhw9WcnTpzQgYGBul+/fu6Kn29l998hEKFvombaYiJfx45WJ7DOmDFjiImJYcWKFTRr1szqOEKIPObs2bOAGeJP9d577xETE8OqVatITk6mefPmV39Ws2ZNfvjhh0z3VaZMGdq2bZvlz0XeZ4vhfS9bvIqcO3v2LEeOHAHgs88+IyoqSgq+ECJTVatWpXz58vTs2ZNFixahteaee+6hVq1a7Nu3j8DAQG677bar2/fo0YM1a9Zkub/q1atz+PBh4uLi3BFfuFg+K5ee75dffqFGjRo89dRTaK0pXry43CxHCJElPz8/lixZgp+fH+3btyc8PJyNGzcCpvefcdW3YsWKUbVq1Sz3FxQUBMD58+dzL7TINbYo+in//dpacnIyo0aNokmTJvj5+fHhhx/KtcRCuJFS5pFW27ambckSZ9vkyaatTx9n2/Hjpi3jzcFq1TLtkZHOtpEjTdvIkc62tD+/GaGhoezbt4/PPvuM48eP06RJE5YuXUpiYiJeKUOlmzZtQil19ZEVed/xbLYo+jExVifIXdHR0TRr1owRI0bw1FNPsW3bNmrVqmV1LCGEB/H19eX5559n165dVKlShb59+xIQEEBsbCxghu23b9/OlClTrrufc+fOAVC0aNFczyxczxZFv149qxPkroCAAC5fvsysWbOYM2cOgYGBVkcSIt/R2jzSWrLEtLVt62zr08e0TZ7sbCtXzrQdP57+9yMjTXvaz/AjR5q2tD39W/mMP2XKFFq1anX1+xIlSjBs2DCOHTvGbbfdxtmzZ4mJiaFQoULUqFGD0qVLX3d/v/32G3feeaesAeKhbFH0g4pZncD1rly5wujRo4mNjSUgIIBNmzbRvXt3q2MJITxMwYIFWbVqVbpz8P/88w/+/v488cQTACxJc34iKioqy32dPn2a7777jscffzz3AotcZYtL9uxm3759dO7cmaioKEJCQujUqdPV825CCHEj2rZtS1BQEE8++SRvvPEG0dHRjBgxgj59+lCtWjU6dOhA//79AfD29r7m1tsJCQls3bqVgwcPMnr0aAIDA/+/vbuPr6I6Ezj+ewJ5IcASCJAQqiAJRQqRNAoIWk0U5MWPSFFBCUoEBKzgohVQGkwAbYu1lG6lIlbDVitrha2oFOSlEkpFDBhA1lhxjcIGRUAsEJJAyLN/zM0lCcG8kHtvMnm+n8/9eO+dMzPPHIc8d86cc4Y5c+YE4lBMPXBFJvnfzwIdQf1QVV588UWuvPJK8vPzeeuttxgzZkygwzLGNGIRERFs3LiR0tJSRo0axWOPPcY999zDwoULAVi+fDl33HEH999/PxkZGTzwwAMV1v/yyy/p378/M2bMoF+/fmzfvt1GDDViopVvUjUyoZ266/2PbGHxTzsFOpSLtmDBAh5//HGSk5N5+eWXianc1dcY43O5ubn07Nkz0GGYJq6681BEdqrqVbXdriua97t1C3QEF0dVERFSUlIICQnhkUceoVmzZoEOyxhjjMu4onm/e1ygI6ib0tJSfvWrXzF69GhUlW7dujF79mxL+MYYY3zCFUm/MTp06BDDhw9n1qxZlJaWUlRUFOiQjDHGuJwrkv6JE4GOoHbWr19Pnz59yMrKYunSpaxcuZIWLVoEOixjjDEu54qkv317oCOouVOnTpGamkpkZCTZ2dlMmTLFprU0xhjjF67oyNeqVaAjqN6BAweIiYkhPDyct99+m9jYWJvRyhhjjF+54kq/oU/D++qrr9K7d2/vuNj4+HhL+MYYY/zOFUm/oSooKGDSpEnceeed9OrVi7FjxwY6JGOMMU2YJX0f+fDDD7nqqqt48cUXmTNnDllZWXTt2jXQYRljjGnCXJH0t/4j0BGcr7CwkFOnTrFx40aefPJJgoODAx2SMaaJWb58OSKCiBAUFESXLl145JFHvI/T9dU+/XWB8/nnn3uPr/Jr+fLlfomhsXFFR77TxYGOwHH06FFWr17NhAkT6NevH/v27SMkJCTQYRljmrjs7GxOnz7N+++/z9y5czl06BAvvfRSoMOqN0uXLuXKSs8fvuyyywIUzfl27drF5s2bmTFjRqBDcUfSHzgw0BHAli1bSElJ4euvv+aGG26ga9eulvCNMQ3CVVc5U7QPHDiQgoIC5s+fzx/+8AdCQ0MDHFn96NGjh/cYG6Jdu3axePHiBpH0XdG8HxYWuH2XlJSQkZFBcnIyLVq0YNu2bXbv3hjTYCUmJnL69GmOHj0a6FBMALgi6QeKqjJixAjmzZtHSkoKO3fuJDExMdBhGWPMBR06dAgRITIyEoD8/HxGjhxJmzZtiI6O5qGHHqK0tBQ4d898165d3H777bRq1YrLL7+cbdu2ebf30Ucfcc011xAWFsaAAQPIy8ursL9jx44xbtw4WrVqRXR0NPPmzaPs6a5JSUlMmTKFvn370q5dO9asWcOAAQOIiIjg9ddfr5fjLS4uZvr06bRr1462bdsyffp0iovP3RPevHkzIsLZs2dZsGABXbt2rXDr48yZM8yePZuoqCgiIyNJTU3l+PHj3uXHjx9n/PjxdOjQgYiICEaNGsXhw4cByMjIQES49957+eKLL7z9DTIyMurl2OpEVRv1KyQ6ThevPKiB8tJLL+kf//jHgO3fGFO/Pvroo0CHUG8yMzPV+TPv2Lt3r/bo0UMHDRrk/e7666/X3r1768aNG3XlypXarl07zczMVFXVvLw8BbR37946bdo03bBhgyYmJmpCQoKqqp45c0a7d++uAwYM0HXr1un8+fM1ODhYu3Tp4t3+TTfdpN///vd19erVunTpUm3VqpX+/Oc/9+67devWumrVKu3Tp482b95cMzMzdciQITps2LBqj68svnfeeeeCZe677z6Njo7WV155RVesWKFRUVE6efJk7/J33nlHAZ08ebL27dtXf/Ob32hubq53+ezZszUqKkpfffVVfeuttzQ2NlbHjBnjXT59+nSNiYnR1atX6xtvvKHx8fE6adIkVVXNz8/X7OxsTU9P106dOml2drZmZ2drfn5+tcdW3XkI7NA65ExX3NM/eNB/+yoqKmLmzJkkJiZy7733Mm7cOP/t3BgTEF0fXRPoEAD4/Jc312m98lN9JyYm8sILLwDORd/YsWO55ppr6NWrFyUlJSxZsoTt27eTmprqXadnz5787ne/A2DOnDnceeedgPMckX379rF27VpiY2MZMmQIOTk5fPDBBwBs3bqV9evXk5OTQ0JCAuBMRT537lwefvhhAO666y5GjRrF6tWriYqKIjU1lby8PLKysmp8fMnJyRU+5+Xl0bVrV/bv388LL7zAqlWrGDlyJAChoaHcfvvtpKWlcckll3jXyc3NZevWrRX6YhUWFrJ48WKee+45Ro8eDcCRI0e47777KCoqIiwsjP3799OnTx9GjBgBQPfu3fnmm28AiImJISYmhr179xISEtIg+h24onm/e3f/7Cc3N5f+/fvzzDPP8Omnn/pnp8YYc5FycnJYs2YNIsKsWbO49NJLAefHwOjRo3n77be5+eabiYqKYvPmzRQWFlZYf/Lkyd73kZGRlJSUALBv3z7atWtHbGysd/l1113nfb9r1y7atGnjTfjgNOkXFBR4/4Z26tTJG0v597Xx/PPPk5OT433FxMQAsGfPHkpLS0lKSqqw/9LSUvbs2VNhG7/+9a/P63z96aefUlxcTGpqqrdpPjU1lTNnzrB//34AJk6cyKZNmxg4cCCzZs0iPz+fgQ2hd/kFuOJKv3Nn325fVcnMzGT69OmEh4ezZs0ahg8f7tudGmMajLpeYTcUCQkJJCQkMGLECBYuXMiYMWMAOHHiBImJiXTs2JGxY8cyd+5cnn322fPWv9Dwt9LSUoKCKl47NmvWrMLnygm87LN67uvXh7i4uAo/LCorH8OF9t+3b9/z1isr89prrxEXF1dhWdkPp1tuuYV//vOfrFu3jqysLIYNG8ZPfvITFi9eXLeD8TFXXOn72o4dO5g4cSL9+/dn9+7dlvCNMY3SnDlzyMnJYcOGDQBs2rSJvLw81q5dy4MPPsjVV19dZStm5UReJjY2lqNHj3qvegH+8Y9zs6UlJCTw7bffVriqzsrKIjw8nO5+aKK94oorCAoKqnCrICsri6CgIK644opq14+LiyMkJISioiLvD6eWLVvy9NNPc+zYMQCeeuopDhw4wNSpU1mxYgXz588nMzOzwnbCwsLOaz0JFFdc6R/9xjfbPXz4MB06dKBv376sW7eOQYMGXfDkN8aYhq5fv37ceOONLFy4kMGDB3t78GdmZhIfH8+SJUt49913azyxzdChQ+nSpQt33303aWlp7Ny5k1WrVtHZ0/x67bXXMnjwYMaMGcNTTz3FV199xeOPP05aWppf5gi49NJLmThxIlOnTqWwsBBV5eGHH2bSpEneK/XvEh4ezkMPPcTMmTNRVTp37kxGRgbHjh0jOjoagI8//pgVK1bw5JNP0qJFC954443zhm0nJiZy5MgRli1bRq9evdi6dSuzZ8/2xSFXry69/xrSKyQ6Th99pn577589e1YXLlyo4eHh+v7779frto0xDZube++rqv7tb39TQLOzs1VV9Wc/+5lGRkZqVFSUpqam6pQpUzQuLk5LSkq8vePz8vK865f1di+Tm5urSUlJGh4eromJiTp79uwKvfe/+eYbHTt2rLZs2VI7duyo6enpevbsWVV1eu+np6erqur48eN1/Pjxqqqanp6u119/fbXHV5Pe+0VFRTpt2jSNiIjQiIgInTZtmhYVFV3weCo7ffq0zpw5Uzt06KCtW7fWW2+9Vb/44osKxzd+/Hjt2LGjhoeH63XXXad79uw5bzvLli3T733ve9q8eXPt3bt3tcfmq977ovV4XyUQQjt11wWLtjDrrk71sr1Dhw5xzz33sH79em677Taef/552rZtWy/bNsY0fLm5ufTs2TPQYZgmrrrzUER2qmqthwO44p5+fO/62c769evp06cPW7ZsYenSpbz22muW8I0xxriGK+7p15dt27bRvn17Nm3aRK9evQIdjjHGGFOvXHGlfzE+++wzb2/TtLQ0srOzLeEbY4xxJVck/a1b67beihUrSEhIYNKkSZw9e5ZmzZrRokWL+g3OGGOMaSBckfRrq6CggAkTJjB27Fji4+NZt26dDcUzxng19g7OpnHz5fnninv6A6+pednDhw/zox/9iE8++YS0tDTS09Np3twV1WCMqQfNmzenpKSE4ODgQIdimqiSkhKf5SVXZLugWkzT3L59e5KTk3n22WfPe0iDMcaEhYVx8uRJG7ljAubEiROEhYX5ZNtNonn/6NGjpKSk8NlnnyEilvCNMRfUoUMHDh8+zKlTp6yZ3/iVqnLq1CmOHDlChw4dfLIPV1zp534Mw+KrXpaVlUVKSgpff/01I0aMoFu3bv4NzhjTqISFhREVFcVXX31FcXFxoMMxTUxoaChRUVE+u9J3RdI/VsXc+yUlJTzxxBMsWLCA2NhY3nvvPRITE/0fnDGm0WnTpg1t2rQJdBjG1DtXNO/36HH+d4sWLWLevHmMGzeOnTt3WsI3xhjT5LniSt/zoCgATp48SatWrXjggQeIjY3ltttuC1xgxhhjTAPiiit9gKKiIqZNm0a/fv0oKCigZcuWlvCNMcaYcvye9EXkBRHZJiJpF1OmvJzsffTv358lS5YwdOhQG3dvjDHGVMGvSV9ERgHNVHUA0E1EutelTHlnTx0n4/6hHDx4kDVr1rBo0SJCQ0N9cwDGGGNMI+bvK/0k4M+e9+uBa+tYxqu08F9c1uNKdu/ezfDhw+spTGOMMcZ9/N0O3hLI97z/BqiqS321ZURkMjDZ87H4kw+37u3cuXM9h2oqaQ8cCXQQLmd17HtWx75ndewfVYxbq56/k/5JoOwxdq2ouqWh2jKqugxYBiAiO1T1qvoP1ZRn9ex7Vse+Z3Xse1bH/iEiO+qynr+b93dyrrm+D/B5HcsYY4wxppb8faX/OvB3EYkBhgF3isgTqpr2HWWu9nOMxhhjjCv59UpfVY/jdNR7D0hW1d2VEn5VZf5VzWaX+SBUcz6rZ9+zOvY9q2Pfszr2jzrVs9hTpIwxxpimwTUz8hljjDHmuzWapO+LmfxMRdXVn4i0EZG1IrJeRP4iIiH+jtENanqeikiUiOT4Ky43qUUd/15EbvFXXG5Sg78XbUXkryKyQ0Se83d8buH5O/D3asrUOPc1iqTvi5n8TEU1rL8UYJGq3gR8BQz1Z4xuUMvz9GnODV81NVTTOhaRHwHRqvqmXwN0gRrW8d3AnzzD91qLiA3jqyURaQv8J878NRcqU6vc1yiSPj6Yyc+cJ4lq6k9Vf6+qGzwfOwBf+yc0V0miBuepiNwAFOD8uDK1k0Q1dSwiwcDzwOcicqv/QnONJKo/j48CvUUkArgEOOCf0FzlLDAGOP4dZZKoRe5rLEm/8ix9UXUsYy6sxvUnIgOAtqr6nj8Cc5lq69lz22Qu8Kgf43KTmpzL9wAfAU8B/URkup9ic4ua1PFWoAvwIJDrKWdqQVWP12AEW61yX2NJ+vUyk5/5TjWqPxFpB/wOmOCnuNymJvX8KPB7Vf3Wb1G5S03q+IfAMlX9CngZSPZTbG5RkzpOB6aq6nzgY+BeP8XW1NQq9zWWxGgz+fletfXnuQJ9DXhMVb/wX2iuUpPzdBDwgIhsBhJE5A/+Cc01alLHnwLdPO+vAux8rp2a1HFbIF5EmgH9ARsf7hu1yn2NYpy+iPwb8HdgE56Z/IA7yk/sU0WZq2vQLGI8aljH9wM/B3Z7vnpWVV/1d6yNWU3quVL5zaqa5L8IG78ansutgRdxmkKDgdtVNb+KzZkq1LCO+wGZOE3824Afq+rJAITb6JX9HRCRHwBjLyb3NYqkD95ejIOBLZ4muTqVMRdm9ecfVs++Z3Xse1bHDUdt/l80mqRvjDHGmIvTWO7pG2OMMeYiWdI3xhhjmghL+sYYnxORZiIigY7DmKbOkr4xLiEiI0Vk4AWWhfnyWQkiMkREflru8y9E5O1yRR4H3vQM36rJ9lI8k0AZY+pR80AHYIypN3OBv4nIIpxx0WUeA7oCd4mI4kzg8YCqPgcgIj8Eiqh+HHUzIAz4UFVPV1r2L2COiHRU1dlAMVDo2f5wYBbOUKOz5VfyTIfbHChW1dJyiyYAp4BbypVtBoQApapaXE2sxpgqWNI3xgVE5FIgHhgB9AOSVXWziCzHSahTgamesptxZvEq8x5Oki6fdENwEnz5Ob+DPN/3wDOZjYiEAgK8D9wM/NYzBr68h4H7VbXsyYxBqlrkWTYGeAYoFJGyRB6M8wOkREQ+L7edYCAc50FET9aoYowxFVjSN8YdxgM7VTXfczVfHe8Vt6qGVl4oIqlAhqp2rWY7C4F/r/Sd94dCuVhuFJFMz/vVwEjP+xWe/65T1SOedVYAEZ4yCUC2qpZ6nh52M8400MaYOrB7+sY0ciLSHJiEc7Ve5h1Pwh0PhInIRhE5ISLf4kzZecFHddZSBhAJBKuqAHHAl8AnOMm9J/AGzu2FIJzWg7vKrd8CuBr4REQGi8hKIBpn3vY5QBYwTEQmADuATp5tGGPqwK70jWn87sW5T19esqpuLvsgIr/Fab4v1ipm5BKRfwdOqerztdlx+YcCeR4H/IrndQLnoTYncX5g7AV+qqrLKq1/EpgmIsuA08BtOC0FG3B+EPxYVdd4+h2MV9XXaxOfMaYiu9I3phHz3Mv/JfD7CywPE5GOOI+SvQsYLyKpItK7UtHBwHWVvgsSkYhyr/Yi0qmKffTxNMm/CfxCVR/Gufcfqqr/59l2OrBERN70xFNZAfAtTr+EG3D6CYwDOotIIs6z2eNs2J8xF8eSvjGN20GchLqz0vdlzfuFwCXAr4AUnPvki4DulcqXUO4+v8clwLFyr8PA2vIFRKQv8AFOB7sEVf1tubi+FZFm6liE8zCQS6n0d0dEhgHbgRtxWgVW4/TeXwUMwXkwTlecUQgrRSS8ukoxxlTNkr4xjZiqlqjqM1UsSvbcY2+Bk5T/CjyjqiNxmtF31GDzX6iqlL1wes9XngdgL9BLVW9V1X2VlvWj3IgAVd3o+c47ckBE5gB/xGmJyMV5Gltr4D9whgH2B3oBiTiPwL0Cp8e/MaYO7J6+MS7laQovaw7fCAwVkT049+4P1HZ7qlqC0yJQ3lrg+u9odS+9wDIRkSDgv4A/q+qnni/74zwP/AXgM1Wd4XlE6wFV/VJEEgD1tCBUbpkwxlTDkr4x7vROufeXAf+NMyueAkvqcT/DPdv0Tq4jIj2AXUA+8KaqPlRW2DNOv2yIYDzObYnTIlJ5sp+WOD8YUsutC+fmChiC07PfGFMLlvSNcaeyyXmCgRJVVRFZj3OvPLq+dqKqp8p/FpEY4E84TfbzgHc9LQ6PqWqhZya/0551d3OBv0Ei8jrwuarOqK9YjTF2T98Yt2jGuX/PwWVfquoZoJWIzAWGAruBF0UkyvMQnAQR6Ykz5K+NiFwuIpfjjIcPLvvsef3AUz6u8s5FJFJEHsK5ws8FHlTVg8AAnHvxe0Vkuoi08V0VGGOqY1f6xrhDGOcmrfHOsOd5AM86nB7xiTi98BcDH+F0ittOxXn336u03cqfgz3bu82z/Rk4QwF/CPwPMEVV/1JW2HMfPgmYjDORzyIReVVVx1VzPOV/xBhj6olUMU+HMcZFRCRKVQ9V+q592bS3F7nta4BBwOue5vrvKhuKM2TwoKr+vZqyG4A8VZ18sTEaY86xpG+MMcY0EdZ8ZowxxjQRlvSNMcaYJsKSvjHGGNNEWNI3xhhjmghL+sYYY0wTYUnfGGOMaSL+HyDleysqE3NYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 继承了上边的图\n",
    "plt.figure(figsize=(8, 6))\n",
    "plt.plot(fpr, tpr, \"b:\", linewidth=2, label=\"SGD\")\n",
    "plot_roc_curve(fpr_forest, tpr_forest, \"Random Forest\")\n",
    "plt.legend(loc=\"lower right\", fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9934248221885138"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rand 比SGD 好很多，ROC AUC的分数也高很多\n",
    "roc_auc_score(y_train_5, y_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9848084544253632"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 再看一下 精度和召回率 也很高\n",
    "y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_5, cv=3)\n",
    "precision_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8251245157719977"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_5, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "* 选择合适的指标利用交叉验证来对分类器进行评估\n",
    "* 选择满足需求的精度/召回率权衡\n",
    "* 使用ROC曲线和ROC AUC分数比较多个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多类别分类器\n",
    "* 尝试5 之外的检测\n",
    "* 多类别分类器 区分两个以上的类别\n",
    "* 随机森里和朴素贝叶斯可以直接处理多个类别\n",
    "* 支持向量机svm和线性分类器只可以处理二元分类器"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 解决方案\n",
    "1. 将数字图片分类0到9，训练10个二元分类器，每个数字一个，检测一张图片时，获取每个分类器的决策分数，哪个最高属于哪个，称为一对多OvA\n",
    "2. 为每一对数字训练一个二元分类器，区分0，1 区分0，2 区分1，2 称为一对一OvO策略，存在N个类别，需要N*（N-1）/2个分类器，最后看哪个类别获胜最多\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 优缺点\n",
    "* OvO 只需要用到部分训练集对其必须区分两个类别进行训练\n",
    "* 对于较小训练集合OvO比较有优势， 大训练集合 OvA 速度快，所以OvA更常用，比如svm 在数据规模扩大时表现糟糕\n",
    "* sklearn 检查到使用二元分类算法进行多类别分类任务，会自动运行OvA，SVM分类器除外"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 预测结果 二元分类器进行多次训练\n",
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-445433.17797054, -487859.08607547, -331607.33639961,\n",
       "        -270945.42809303, -381448.05810337,   45693.37296594,\n",
       "        -639686.00233291, -418236.06305131, -594450.41159686,\n",
       "        -487196.91384615]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 内部实际上训练了10个二元分类器，获得图片的决策分数，然后选择了分数最高的类别\n",
    "# 返回10个分数，每个类别1个\n",
    "some_digit_scores = sgd_clf.decision_function([some_digit])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 拿到分数最高的索引\n",
    "np.argmax(some_digit_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 目标类别列表会存储在classes_这个属性中，按值大小排列，\n",
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 通过索引 来拿到分类标签中对应的值\n",
    "sgd_clf.classes_[np.argmax(some_digit_scores)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用OvO策略，一对一或者一对多\n",
    "from sklearn.multiclass import OneVsOneClassifier \n",
    "# 把二元分类器，封装成 ovo\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5, tol=-np.infty, random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)  # 训练\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)   # 多个ovo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用随机森林\n",
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.2, 0. , 0. , 0. , 0.1, 0.7, 0. , 0. , 0. , 0. ]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机森林直接将实例分为多个类别，调用predict_proba()可以获得分类器将每个实例分类为每个类别的概率列表\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.86842631, 0.85554278, 0.84862729])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用交叉验证评估SGD的准确率\n",
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\numpy\\core\\_methods.py:38: RuntimeWarning: overflow encountered in reduce\n",
      "  return umr_sum(a, axis, dtype, out, keepdims, initial, where)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\numpy\\core\\_methods.py:183: RuntimeWarning: overflow encountered in reduce\n",
      "  arrmean = umr_sum(arr, axis, dtype, keepdims=True)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\numpy\\core\\_methods.py:199: RuntimeWarning: overflow encountered in reduce\n",
      "  ret = umr_sum(x, axis, dtype, out, keepdims)\n",
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\preprocessing\\data.py:694: RuntimeWarning: invalid value encountered in true_divide\n",
      "  X /= self.scale_\n"
     ]
    }
   ],
   "source": [
    "# 将输入进行简单缩放 ，可以得到准确率 90 %以上\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "# 实例化\n",
    "scaler = StandardScaler()\n",
    "# 数据缩放\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float16))\n",
    "# 再次评估\n",
    "#cross_val_score(sgd_clf, X_train_scaled, y_train, cv=3, scoring=\"accuracy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 错误分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 项目流程\n",
    "1. 探索数据准备的选项\n",
    "2. 尝试多个模型\n",
    "3. 选择最佳模型并用GridSearchCV对参数进行微调\n",
    "4. 尽可能自动化\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 确定了一个相对合适的模型，进一步优化，分析其错误类型\n",
    "* 查看混淆矩阵\n",
    "* 使用cross_val_predict()进行预测\n",
    "* 调用confusion_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDClassifier'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  \"and default tol will be 1e-3.\" % type(self), FutureWarning)\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 239. MiB for an array with shape (39996, 784) and data type float64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-75-90d92e22fed9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my_train_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcross_val_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msgd_clf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train_scaled\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mconf_mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mconf_mx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36mcross_val_predict\u001b[1;34m(estimator, X, y, groups, cv, n_jobs, verbose, fit_params, pre_dispatch, method)\u001b[0m\n\u001b[0;32m    678\u001b[0m     prediction_blocks = parallel(delayed(_fit_and_predict)(\n\u001b[0;32m    679\u001b[0m         clone(estimator), X, y, train, test, verbose, fit_params, method)\n\u001b[1;32m--> 680\u001b[1;33m         for train, test in cv.split(X, y, groups))\n\u001b[0m\u001b[0;32m    681\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    682\u001b[0m     \u001b[1;31m# Concatenate the predictions\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    777\u001b[0m             \u001b[1;31m# was dispatched. In particular this covers the edge\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    778\u001b[0m             \u001b[1;31m# case of Parallel used with an exhausted iterator.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 779\u001b[1;33m             \u001b[1;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    780\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    781\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    623\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    624\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 625\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    626\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    586\u001b[0m         \u001b[0mdispatch_timestamp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    587\u001b[0m         \u001b[0mcb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBatchCompletionCallBack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdispatch_timestamp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 588\u001b[1;33m         \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    589\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    590\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    109\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    110\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 111\u001b[1;33m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    112\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    113\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    330\u001b[0m         \u001b[1;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    331\u001b[0m         \u001b[1;31m# arguments in memory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 332\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    333\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    334\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    129\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    130\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 131\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    132\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    133\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\u001b[0m in \u001b[0;36m_fit_and_predict\u001b[1;34m(estimator, X, y, train, test, verbose, fit_params, method)\u001b[0m\n\u001b[0;32m    751\u001b[0m         \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    752\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 753\u001b[1;33m         \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    754\u001b[0m     \u001b[0mfunc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    755\u001b[0m     \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, coef_init, intercept_init, sample_weight)\u001b[0m\n\u001b[0;32m    584\u001b[0m                          \u001b[0mloss\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearning_rate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearning_rate\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    585\u001b[0m                          \u001b[0mcoef_init\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcoef_init\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mintercept_init\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mintercept_init\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m                          sample_weight=sample_weight)\n\u001b[0m\u001b[0;32m    587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, alpha, C, loss, learning_rate, coef_init, intercept_init, sample_weight)\u001b[0m\n\u001b[0;32m    416\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclasses_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    417\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 418\u001b[1;33m         \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_X_y\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"C\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    419\u001b[0m         \u001b[0mn_samples\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_features\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    420\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_X_y\u001b[1;34m(X, y, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    571\u001b[0m     X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,\n\u001b[0;32m    572\u001b[0m                     \u001b[0mensure_2d\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mallow_nd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mensure_min_samples\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 573\u001b[1;33m                     ensure_min_features, warn_on_dtype, estimator)\n\u001b[0m\u001b[0;32m    574\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mmulti_output\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    575\u001b[0m         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,\n",
      "\u001b[1;32mc:\\users\\administrator\\appdata\\local\\programs\\python\\python37-32\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    431\u001b[0m                                       force_all_finite)\n\u001b[0;32m    432\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 433\u001b[1;33m         \u001b[0marray\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0morder\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    434\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    435\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mensure_2d\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 239. MiB for an array with shape (39996, 784) and data type float64"
     ]
    }
   ],
   "source": [
    "y_train_pred = cross_val_predict(sgd_clf, X_train_scaled, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "不更新硬件配置 从代码上如何解决这类问题  有没有方法解决此问题"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本周作业我先不纠结了，先提交解锁下一周课程跟上大伙的步伐吧 !!!! 老师辛苦了 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
